Python presto_installer.StandalonePrestoInstaller类代码示例

 def assert_common_configs(self, host):
     installer = StandalonePrestoInstaller(self)
     installer.assert_installed(self, host)
     self.assert_file_content(host, '/etc/presto/jvm.config',
                              self.default_jvm_config_)
     self.assert_node_config(host, self.default_node_properties_)
     self.assert_has_default_connector(host)

    def test_install_when_connector_json_exists(self):
        installer = StandalonePrestoInstaller(self, dummy=True)
        topology = {"coordinator": "master",
                    "workers": ["slave1"]}
        self.upload_topology(topology)
        self.cluster.write_content_to_host(
            'connector.name=jmx',
            os.path.join(constants.CONNECTORS_DIR, 'jmx.properties'),
            self.cluster.master
        )

        cmd_output = installer.install()
        expected = ['Deploying rpm on master...',
                    'Deploying rpm on slave1...',
                    'Package deployed successfully on: slave1',
                    'Package installed successfully on: slave1',
                    'Package deployed successfully on: master',
                    'Package installed successfully on: master',
                    'Deploying configuration on: master',
                    'Deploying jmx.properties, tpch.properties '
                    'connector configurations on: master ',
                    'Deploying configuration on: slave1',
                    'Deploying jmx.properties, tpch.properties '
                    'connector configurations on: slave1 ']

        actual = cmd_output.splitlines()
        self.assertEqual(sorted(expected), sorted(actual))

        for container in [self.cluster.master,
                          self.cluster.slaves[0]]:
            installer.assert_installed(self, container)
            self.assert_has_default_config(container)
            self.assert_has_default_connector(container)
            self.assert_has_jmx_connector(container)

    def test_catalog_add_lost_host(self):
        installer = StandalonePrestoInstaller(self)
        self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER)
        self.upload_topology()
        installer.install()
        self.run_prestoadmin('catalog remove tpch')

        self.cluster.stop_host(
            self.cluster.slaves[0])
        self.cluster.write_content_to_host(
            'connector.name=tpch',
            os.path.join(get_catalog_directory(), 'tpch.properties'),
            self.cluster.master
        )
        output = self.run_prestoadmin('catalog add tpch', raise_error=False)
        for host in self.cluster.all_internal_hosts():
            deploying_message = 'Deploying tpch.properties catalog configurations on: %s'
            self.assertTrue(deploying_message % host in output,
                            'expected %s \n actual %s'
                            % (deploying_message % host, output))
        self.assertRegexpMatches(
            output,
            self.down_node_connection_error(self.cluster.internal_slaves[0])
        )
        self.assertEqual(len(output.splitlines()),
                         len(self.cluster.all_hosts()) +
                         self.len_down_node_error)
        self.run_prestoadmin('server start', raise_error=False)

        for host in [self.cluster.master,
                     self.cluster.slaves[1],
                     self.cluster.slaves[2]]:
            self.assert_has_default_catalog(host)
        self._assert_catalogs_loaded([['system'], ['tpch']])

    def test_connection_to_coord_lost(self):
        installer = StandalonePrestoInstaller(self)
        down_node = self.cluster.internal_slaves[0]
        topology = {"coordinator": down_node,
                    "workers": [self.cluster.internal_master,
                                self.cluster.internal_slaves[1],
                                self.cluster.internal_slaves[2]]}
        self.upload_topology(topology=topology)
        self.cluster.stop_host(
            self.cluster.slaves[0])

        actual_out = installer.install(
            coordinator=down_node, pa_raise_error=False)

        self.assertRegexpMatches(
            actual_out,
            self.down_node_connection_error(down_node)
        )

        for host in [self.cluster.master,
                     self.cluster.slaves[1],
                     self.cluster.slaves[2]]:
            self.assert_common_configs(host)
            self.assert_file_content(
                host,
                '/etc/presto/config.properties',
                self.default_workers_config_with_slave1_
            )

    def test_connector_add_lost_host(self):
        installer = StandalonePrestoInstaller(self)
        self.setup_cluster(self.PA_ONLY_CLUSTER)
        self.upload_topology()
        installer.install()
        self.run_prestoadmin('connector remove tpch')

        self.cluster.stop_host(
            self.cluster.slaves[0])
        self.cluster.write_content_to_host(
            'connector.name=tpch',
            os.path.join(constants.CONNECTORS_DIR, 'tpch.properties'),
            self.cluster.master
        )
        output = self.run_prestoadmin('connector add tpch')
        for host in self.cluster.all_internal_hosts():
            deploying_message = 'Deploying tpch.properties connector ' \
                                'configurations on: %s'
            self.assertTrue(deploying_message % host in output,
                            'expected %s \n actual %s'
                            % (deploying_message % host, output))
        self.assertRegexpMatches(
            output,
            self.down_node_connection_error(self.cluster.internal_slaves[0])
        )
        self.assertEqual(len(output.splitlines()),
                         len(self.cluster.all_hosts()) +
                         self.len_down_node_error)
        self.run_prestoadmin('server start')

        for host in [self.cluster.master,
                     self.cluster.slaves[1],
                     self.cluster.slaves[2]]:
            self.assert_has_default_connector(host)
        self._assert_connectors_loaded([['system'], ['tpch']])

 def setUp(self):
     super(TestServerUpgrade, self).setUp()
     self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER)
     self.dummy_installer = StandalonePrestoInstaller(
         self, (os.path.join(prestoadmin.main_dir, 'tests', 'product',
                             'resources'), 'dummy-rpm.rpm'))
     self.real_installer = StandalonePrestoInstaller(self)

    def test_install_interactive_with_hostnames(self):
        installer = StandalonePrestoInstaller(self)
        self.cluster.write_content_to_host(
            'connector.name=jmx',
            os.path.join(constants.CONNECTORS_DIR, 'jmx.properties'),
            self.cluster.master
        )
        rpm_name = installer.copy_presto_rpm_to_master()
        self.write_test_configs(self.cluster)

        cmd_output = self.run_script_from_prestoadmin_dir(
            'echo -e "root\n22\n%(master)s\n%(slave1)s\n" | '
            './presto-admin server install /mnt/presto-admin/%(rpm)s ',
            rpm=rpm_name)

        actual = cmd_output.splitlines()
        expected = [r'Enter user name for SSH connection to all nodes: '
                    r'\[root\] '
                    r'Enter port number for SSH connections to all nodes: '
                    r'\[22\] '
                    r'Enter host name or IP address for coordinator node. '
                    r'Enter an external host name or ip address if this is a '
                    r'multi-node cluster: \[localhost\] '
                    r'Enter host names or IP addresses for worker nodes '
                    r'separated by spaces: '
                    r'\[localhost\] Using rpm_specifier as a local path',
                    r'Package deployed successfully on: ' +
                    self.cluster.internal_master,
                    r'Package installed successfully on: ' +
                    self.cluster.internal_master,
                    r'Package deployed successfully on: ' +
                    self.cluster.internal_slaves[0],
                    r'Package installed successfully on: ' +
                    self.cluster.internal_slaves[0],
                    r'Deploying configuration on: ' +
                    self.cluster.internal_master,
                    r'Deploying jmx.properties, tpch.properties connector '
                    r'configurations on: ' +
                    self.cluster.internal_master,
                    r'Deploying configuration on: ' +
                    self.cluster.internal_slaves[0],
                    r'Deploying jmx.properties, tpch.properties connector '
                    r'configurations on: ' +
                    self.cluster.internal_slaves[0],
                    r'Deploying rpm on .*\.\.\.',
                    r'Deploying rpm on .*\.\.\.',
                    r'Fetching local presto rpm at path: .*',
                    r'Found existing rpm at: .*'
                    ]

        self.assertRegexpMatchesLineByLine(actual, expected)
        for container in [self.cluster.master,
                          self.cluster.slaves[0]]:
            installer.assert_installed(self, container)
            self.assert_has_default_config(container)
            self.assert_has_default_connector(container)
            self.assert_has_jmx_connector(container)

    def test_install_interactive_with_ips(self):
        installer = StandalonePrestoInstaller(self)
        ips = self.cluster.get_ip_address_dict()
        rpm_name = installer.copy_presto_rpm_to_master()
        self.write_test_configs(self.cluster)

        additional_keywords = {
            'rpm': rpm_name,
            'master_ip': ips[self.cluster.master],
            'slave1_ip': ips[self.cluster.slaves[0]]
        }
        cmd_output = self.run_script_from_prestoadmin_dir(
            'echo -e "root\n22\n%(master_ip)s\n%(slave1_ip)s\n" | '
            './presto-admin server install /mnt/presto-admin/%(rpm)s ',
            **additional_keywords).splitlines()
        expected = [r'Enter user name for SSH connection to all nodes: '
                    r'\[root\] '
                    r'Enter port number for SSH connections to all nodes: '
                    r'\[22\] '
                    r'Enter host name or IP address for coordinator node. '
                    r'Enter an external host name or ip address if this is a '
                    r'multi-node cluster: \[localhost\] '
                    r'Enter host names or IP addresses for worker nodes '
                    r'separated by spaces: '
                    r'\[localhost\] Using rpm_specifier as a local path',
                    r'Package deployed successfully on: ' +
                    ips[self.cluster.master],
                    r'Package installed successfully on: ' +
                    ips[self.cluster.master],
                    r'Package deployed successfully on: ' +
                    ips[self.cluster.slaves[0]],
                    r'Package installed successfully on: ' +
                    ips[self.cluster.slaves[0]],
                    r'Deploying configuration on: ' +
                    ips[self.cluster.master],
                    r'Deploying tpch.properties connector '
                    r'configurations on: ' +
                    ips[self.cluster.master] + r' ',
                    r'Deploying configuration on: ' +
                    ips[self.cluster.slaves[0]],
                    r'Deploying tpch.properties connector '
                    r'configurations on: ' +
                    ips[self.cluster.slaves[0]] + r' ',
                    r'Deploying rpm on .*\.\.\.',
                    r'Deploying rpm on .*\.\.\.',
                    r'Fetching local presto rpm at path: .*',
                    r'Found existing rpm at: .*']

        cmd_output.sort()
        expected.sort()
        for expected_regexp, actual_line in zip(expected, cmd_output):
            self.assertRegexpMatches(actual_line, expected_regexp)

        self.assert_installed_with_regex_configs(
            self.cluster.master,
            [self.cluster.slaves[0]])

 def test_system_info_pa_separate_node(self):
     installer = StandalonePrestoInstaller(self)
     self.setup_cluster(NoHadoopBareImageProvider(), self.PA_ONLY_CLUSTER)
     topology = {"coordinator": "slave1",
                 "workers": ["slave2", "slave3"]}
     self.upload_topology(topology=topology)
     installer.install(coordinator='slave1')
     self.test_basic_system_info(
         coordinator=self.cluster.internal_slaves[0],
         hosts=self.cluster.slaves)

 def test_start_stop_restart_worker_down(self):
     installer = StandalonePrestoInstaller(self)
     self.setup_cluster(NoHadoopBareImageProvider(), self.PA_ONLY_CLUSTER)
     topology = {"coordinator": "slave1",
                 "workers": ["master", "slave2", "slave3"]}
     self.upload_topology(topology=topology)
     installer.install(coordinator='slave1')
     self.assert_start_stop_restart_down_node(
         self.cluster.slaves[0],
         self.cluster.internal_slaves[0])

class TestServerUninstall(BaseProductTestCase):
    def setUp(self):
        super(TestServerUninstall, self).setUp()
        self.installer = StandalonePrestoInstaller(self)

    @attr('smoketest')
    def test_uninstall(self):
        self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER)
        start_output = self.run_prestoadmin('server start')
        process_per_host = self.get_process_per_host(start_output.splitlines())
        self.assert_started(process_per_host)

        cmd_output = self.run_prestoadmin(
            'server uninstall', raise_error=False).splitlines()
        self.assert_stopped(process_per_host)
        expected = uninstall_output + self.expected_stop()[:]
        self.assertRegexpMatchesLineByLine(cmd_output, expected)

        for container in self.cluster.all_hosts():
            self.assert_uninstalled_dirs_removed(container)

    def assert_uninstalled_dirs_removed(self, container):
        self.installer.assert_uninstalled(container)
        self.assert_path_removed(container, '/etc/presto')
        self.assert_path_removed(container, '/usr/lib/presto')
        self.assert_path_removed(container, '/var/lib/presto')
        self.assert_path_removed(container, '/usr/shared/doc/presto')
        self.assert_path_removed(container, '/etc/init.d/presto')

    def test_uninstall_twice(self):
        self.test_uninstall()

        output = self.run_prestoadmin('server uninstall', raise_error=False)
        with open(os.path.join(LOCAL_RESOURCES_DIR, 'uninstall_twice.txt'),
                  'r') as f:
            expected = f.read()

        self.assertEqualIgnoringOrder(expected, output)

    def test_uninstall_lost_host(self):
        self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER)

        self.cluster.stop_host(
            self.cluster.slaves[0])

        expected = self.down_node_connection_error(
            self.cluster.internal_slaves[0])
        cmd_output = self.run_prestoadmin('server uninstall',
                                          raise_error=False)
        self.assertRegexpMatches(cmd_output, expected)

        for container in [self.cluster.internal_master,
                          self.cluster.internal_slaves[1],
                          self.cluster.internal_slaves[2]]:
            self.assert_uninstalled_dirs_removed(container)

 def test_system_info_pa_separate_node(self):
     installer = StandalonePrestoInstaller(self)
     self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER)
     topology = {"coordinator": "slave1",
                 "workers": ["slave2", "slave3"]}
     self.upload_topology(topology=topology)
     installer.install(coordinator='slave1')
     self.run_prestoadmin('server start')
     actual = self.run_prestoadmin('collect system_info')
     self._test_basic_system_info(
         actual,
         coordinator=self.cluster.internal_slaves[0],
         hosts=self.cluster.slaves)

 def test_install_with_wrong_topology(self):
     installer = StandalonePrestoInstaller(self)
     rpm_name = installer.copy_presto_rpm_to_master()
     topology = {'coordinator': 'dummy_master', 'workers': ['slave1']}
     self.upload_topology(topology)
     expected = 'u\'dummy_master\' is not a valid ip address or' \
                ' host name.' \
                '  More detailed information can be found in ' \
                '/var/log/prestoadmin/presto-admin.log\n'
     self.assertRaisesRegexp(OSError,
                             expected,
                             self.run_prestoadmin,
                             'server install /mnt/presto-admin/%(rpm)s ',
                             rpm=rpm_name)

    def test_install_twice(self):
        installer = StandalonePrestoInstaller(self)
        self.upload_topology()
        cmd_output = installer.install()
        expected = self.format_err_msgs_with_internal_hosts(installed_all_hosts_output)

        actual = cmd_output.splitlines()
        self.assertRegexpMatchesLineByLine(actual, expected)

        for container in self.cluster.all_hosts():
            installer.assert_installed(self, container)
            self.assert_has_default_config(container)
            self.assert_has_default_connector(container)

        output = installer.install(pa_raise_error=False)

        self.default_keywords.update(installer.get_keywords())

        with open(os.path.join(LOCAL_RESOURCES_DIR, 'install_twice.txt'),
                  'r') as f:
            expected = f.read()
        expected = self.escape_for_regex(
            self.replace_keywords(expected))

        self.assertRegexpMatchesLineByLine(output.splitlines(),
                                           expected.splitlines())
        for container in self.cluster.all_hosts():
            installer.assert_installed(self, container)
            self.assert_has_default_config(container)
            self.assert_has_default_connector(container)

    def test_install_with_malformed_topology(self):
        installer = StandalonePrestoInstaller(self)
        rpm_name = installer.copy_presto_rpm_to_master()
        topology = {'coordinator': 'master',
                    'workers': 'slave1' 'slave2'}
        self.upload_topology(topology)
        expected = 'Workers must be of type list.  Found <type \'unicode\'>.' \
                   '  More detailed information can be found in ' \
                   '/var/log/prestoadmin/presto-admin.log'

        self.assertRaisesRegexp(OSError,
                                expected,
                                self.run_prestoadmin,
                                'server install /mnt/presto-admin/%(rpm)s ',
                                rpm=rpm_name)