Python RemoteHelper.collect_info方法代码示例

# 需要导入模块: from perfrunner.helpers.remote import RemoteHelper [as 别名]
# 或者: from perfrunner.helpers.remote.RemoteHelper import collect_info [as 别名]
def main():
    options, args = get_options()

    cluster_spec = ClusterSpec()
    cluster_spec.parse(options.cluster_spec_fname, args)

    remote = RemoteHelper(cluster_spec, test_config=None, verbose=False)

    remote.collect_info()
    for hostname in cluster_spec.yield_hostnames():
        for fname in glob.glob('{}/*.zip'.format(hostname)):
            shutil.move(fname, '{}.zip'.format(hostname))

# 需要导入模块: from perfrunner.helpers.remote import RemoteHelper [as 别名]
# 或者: from perfrunner.helpers.remote.RemoteHelper import collect_info [as 别名]
def main():
    args = get_args()

    cluster_spec = ClusterSpec()
    cluster_spec.parse(args.cluster_spec_fname)

    remote = RemoteHelper(cluster_spec, verbose=False)

    remote.collect_info()
    for hostname in cluster_spec.servers:
        for fname in glob.glob('{}/*.zip'.format(hostname)):
            shutil.move(fname, '{}.zip'.format(hostname))

    if cluster_spec.backup is not None:
        logs = os.path.join(cluster_spec.backup, 'logs')
        if os.path.exists(logs):
            shutil.make_archive('tools', 'zip', logs)

# 需要导入模块: from perfrunner.helpers.remote import RemoteHelper [as 别名]
# 或者: from perfrunner.helpers.remote.RemoteHelper import collect_info [as 别名]

#.........这里部分代码省略.........
        self.rest = RestHelper(cluster_spec)
        self.remote = RemoteHelper(cluster_spec)

        if experiment:
            self.experiment = ExperimentHelper(experiment,
                                               cluster_spec, test_config)

        self.master_node = cluster_spec.yield_masters().next()
        self.build = self.rest.get_version(self.master_node)

        self.cbagent = CbAgent(self)
        self.metric_helper = MetricHelper(self)
        self.reporter = Reporter(self)
        self.reports = {}
        self.snapshots = []
        self.master_events = []

        if self.test_config.test_case.use_workers:
            self.worker_manager = WorkerManager(cluster_spec, test_config)

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        if self.test_config.test_case.use_workers:
            self.worker_manager.terminate()
        if exc_type != exc.KeyboardInterrupt:
            self.debug()
        for master in self.cluster_spec.yield_masters():
            if not self.rest.is_balanced(master):
                logger.interrupt('Rebalance failed')

            num_failovers = self.rest.get_failover_counter(master)
            if hasattr(self, 'rebalance_settings'):
                if self.rebalance_settings.failover or \
                        self.rebalance_settings.graceful_failover:
                    continue
            if num_failovers:
                logger.interrupt(
                    'Failover happened {} time(s)'.format(num_failovers)
                )

    def compact_bucket(self):
        for master in self.cluster_spec.yield_masters():
            for bucket in self.test_config.buckets:
                self.rest.trigger_bucket_compaction(master, bucket)
        for master in self.cluster_spec.yield_masters():
            self.monitor.monitor_task(master, 'bucket_compaction')

    def wait_for_persistence(self):
        for master in self.cluster_spec.yield_masters():
            for bucket in self.test_config.buckets:
                self.monitor.monitor_disk_queue(master, bucket)
                self.monitor.monitor_tap_replication(master, bucket)

    def load(self):
        load_settings = self.test_config.load_settings
        log_phase('load phase', load_settings)
        self.worker_manager.run_workload(load_settings, self.target_iterator)
        self.worker_manager.wait_for_workers()

    def hot_load(self):
        hot_load_settings = self.test_config.hot_load_settings
        log_phase('hot load phase', hot_load_settings)
        self.worker_manager.run_workload(hot_load_settings,
                                         self.target_iterator)
        self.worker_manager.wait_for_workers()

    def access(self):
        access_settings = self.test_config.access_settings
        log_phase('access phase', access_settings)
        self.worker_manager.run_workload(access_settings, self.target_iterator)
        self.worker_manager.wait_for_workers()

    def access_bg(self):
        access_settings = self.test_config.access_settings
        log_phase('access in background', access_settings)
        self.worker_manager.run_workload(access_settings, self.target_iterator,
                                         timer=access_settings.time)

    def access_bg_with_ddocs(self):
        access_settings = self.test_config.access_settings
        log_phase('access phase', access_settings)
        index_type = self.test_config.index_settings.index_type
        self.worker_manager.run_workload(access_settings, self.target_iterator,
                                         timer=access_settings.time,
                                         ddocs=self.ddocs,
                                         index_type=index_type)

    def timer(self):
        access_settings = self.test_config.access_settings
        logger.info('Running phase for {} seconds'.format(access_settings.time))
        time.sleep(access_settings.time)

    def debug(self):
        self.remote.collect_info()
        for hostname in self.cluster_spec.yield_hostnames():
            for fname in glob.glob('{}/*.zip'.format(hostname)):
                shutil.move(fname, '{}.zip'.format(hostname))
        self.reporter.save_web_logs()

# 需要导入模块: from perfrunner.helpers.remote import RemoteHelper [as 别名]
# 或者: from perfrunner.helpers.remote.RemoteHelper import collect_info [as 别名]

#.........这里部分代码省略.........

    def __exit__(self, exc_type, exc_val, exc_tb):
        if self.test_config.test_case.use_workers:
            self.worker_manager.terminate()
        if exc_type != exc.KeyboardInterrupt and '--nodebug' not in sys.argv:
            self.debug()

            self.check_core_dumps()
            for master in self.cluster_spec.yield_masters():
                if not self.rest.is_balanced(master):
                    logger.interrupt('Rebalance failed')
                self.check_failover(master)

    def check_failover(self, master):
        if hasattr(self, 'rebalance_settings'):
            if self.rebalance_settings.failover or \
                    self.rebalance_settings.graceful_failover:
                return

        num_failovers = self.rest.get_failover_counter(master)
        if num_failovers:
            logger.interrupt(
                'Failover happened {} time(s)'.format(num_failovers)
            )

    def check_core_dumps(self):
        dumps_per_host = self.remote.detect_core_dumps()
        core_dumps = {
            host: dumps for host, dumps in dumps_per_host.items() if dumps
        }
        if core_dumps:
            logger.interrupt(pretty_dict(core_dumps))

    def compact_bucket(self):
        for master in self.cluster_spec.yield_masters():
            for bucket in self.test_config.buckets:
                self.rest.trigger_bucket_compaction(master, bucket)
        time.sleep(self.MONITORING_DELAY)
        for master in self.cluster_spec.yield_masters():
            self.monitor.monitor_task(master, 'bucket_compaction')

    def wait_for_persistence(self):
        for master in self.cluster_spec.yield_masters():
            for bucket in self.test_config.buckets:
                self.monitor.monitor_disk_queues(master, bucket)
                self.monitor.monitor_tap_queues(master, bucket)
                self.monitor.monitor_upr_queues(master, bucket)

    def load(self):
        load_settings = self.test_config.load_settings
        log_phase('load phase', load_settings)
        self.worker_manager.run_workload(load_settings, self.target_iterator)
        self.worker_manager.wait_for_workers()

    def hot_load(self):
        hot_load_settings = self.test_config.hot_load_settings
        log_phase('hot load phase', hot_load_settings)
        self.worker_manager.run_workload(hot_load_settings,
                                         self.target_iterator)
        self.worker_manager.wait_for_workers()

    def access(self):
        access_settings = self.test_config.access_settings
        log_phase('access phase', access_settings)
        self.worker_manager.run_workload(access_settings, self.target_iterator)
        self.worker_manager.wait_for_workers()

    def access_bg(self):
        access_settings = self.test_config.access_settings
        log_phase('access phase in background', access_settings)
        access_settings.index_type = self.test_config.index_settings.index_type
        access_settings.n1ql = getattr(self, 'n1ql', None)
        access_settings.ddocs = getattr(self, 'ddocs', None)
        self.worker_manager.run_workload(access_settings, self.target_iterator,
                                         timer=access_settings.time)

    def timer(self):
        access_settings = self.test_config.access_settings
        logger.info('Running phase for {} seconds'.format(access_settings.time))
        time.sleep(access_settings.time)

    def debug(self):
        self.remote.collect_info()
        for hostname in self.cluster_spec.yield_hostnames():
            for fname in glob.glob('{}/*.zip'.format(hostname)):
                shutil.move(fname, '{}.zip'.format(hostname))

        self.reporter.save_web_logs()

        if self.test_config.cluster.run_cbq:
            self.remote.collect_cbq_logs()
            for hostname in self.cluster_spec.yield_hostnames():
                for fname in glob.glob('{}/cbq.log'.format(hostname)):
                    shutil.move(fname, '{}-cbq.log'.format(hostname))

        if self.cluster_spec.gateways:
            self.remote.collect_info_gateway()
            self.remote.collect_info_gateload()
            self.reporter.check_sgw_logs()
            self.reporter.save_expvar()

# 需要导入模块: from perfrunner.helpers.remote import RemoteHelper [as 别名]
# 或者: from perfrunner.helpers.remote.RemoteHelper import collect_info [as 别名]
class PerfTest:

    COLLECTORS = {}

    ROOT_CERTIFICATE = 'root.pem'

    def __init__(self,
                 cluster_spec: ClusterSpec,
                 test_config: TestConfig,
                 verbose: bool):
        self.cluster_spec = cluster_spec
        self.test_config = test_config

        self.target_iterator = TargetIterator(cluster_spec, test_config)

        self.cluster = ClusterManager(cluster_spec, test_config)
        self.memcached = MemcachedHelper(test_config)
        self.monitor = Monitor(cluster_spec, test_config, verbose)
        self.rest = RestHelper(cluster_spec)
        self.remote = RemoteHelper(cluster_spec, verbose)
        self.profiler = Profiler(cluster_spec, test_config)

        self.master_node = next(cluster_spec.masters)
        self.build = self.rest.get_version(self.master_node)

        self.metrics = MetricHelper(self)
        self.reporter = ShowFastReporter(cluster_spec, test_config, self.build)

        self.cbmonitor_snapshots = []
        self.cbmonitor_clusters = []

        if self.test_config.test_case.use_workers:
            self.worker_manager = WorkerManager(cluster_spec, test_config,
                                                verbose)

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        failure = self.debug()

        self.tear_down()

        if exc_type == KeyboardInterrupt:
            logger.warn('The test was interrupted')
            return True

        if failure:
            logger.interrupt(failure)

    @property
    def query_nodes(self) -> List[str]:
        return self.rest.get_active_nodes_by_role(self.master_node, 'n1ql')

    @property
    def index_nodes(self) -> List[str]:
        return self.rest.get_active_nodes_by_role(self.master_node, 'index')

    @property
    def fts_nodes(self) -> List[str]:
        return self.rest.get_active_nodes_by_role(self.master_node, 'fts')

    @property
    def analytics_nodes(self) -> List[str]:
        return self.rest.get_active_nodes_by_role(self.master_node, 'cbas')

    @property
    def eventing_nodes(self) -> List[str]:
        return self.rest.get_active_nodes_by_role(self.master_node, 'eventing')

    def tear_down(self):
        if self.test_config.test_case.use_workers:
            self.worker_manager.download_celery_logs()
            self.worker_manager.terminate()

        if self.test_config.cluster.online_cores:
            self.remote.enable_cpu()

        if self.test_config.cluster.kernel_mem_limit:
            self.collect_logs()

            self.cluster.reset_memory_settings()

    def collect_logs(self):
        self.remote.collect_info()

        for hostname in self.cluster_spec.servers:
            for fname in glob.glob('{}/*.zip'.format(hostname)):
                shutil.move(fname, '{}.zip'.format(hostname))

    def reset_memory_settings(self):
        if self.test_config.cluster.kernel_mem_limit:
            for service in self.test_config.cluster.kernel_mem_limit_services:
                for server in self.cluster_spec.servers_by_role(service):
                    self.remote.reset_memory_settings(host_string=server)
            self.monitor.wait_for_servers()

    def debug(self) -> str:
        failure = self.check_core_dumps()
        failure = self.check_rebalance() or failure
#.........这里部分代码省略.........