Python SqoopHook.import_query方法代码示例

# 需要导入模块: from airflow.contrib.hooks.sqoop_hook import SqoopHook [as 别名]
# 或者: from airflow.contrib.hooks.sqoop_hook.SqoopHook import import_query [as 别名]
    def execute(self, context):
        """
        Execute sqoop job
        """
        hook = SqoopHook(conn_id=self.conn_id,
                         verbose=self.verbose,
                         num_mappers=self.num_mappers,
                         hcatalog_database=self.hcatalog_database,
                         hcatalog_table=self.hcatalog_table,
                         properties=self.properties)

        if self.cmd_type == 'export':
            hook.export_table(
                table=self.table,
                export_dir=self.export_dir,
                input_null_string=self.input_null_string,
                input_null_non_string=self.input_null_non_string,
                staging_table=self.staging_table,
                clear_staging_table=self.clear_staging_table,
                enclosed_by=self.enclosed_by,
                escaped_by=self.escaped_by,
                input_fields_terminated_by=self.input_fields_terminated_by,
                input_lines_terminated_by=self.input_lines_terminated_by,
                input_optionally_enclosed_by=self.input_optionally_enclosed_by,
                batch=self.batch,
                relaxed_isolation=self.relaxed_isolation)
        elif self.cmd_type == 'import':
            if not self.table:
                hook.import_table(
                    table=self.table,
                    target_dir=self.target_dir,
                    append=self.append,
                    file_type=self.file_type,
                    columns=self.columns,
                    split_by=self.split_by,
                    where=self.where,
                    direct=self.direct,
                    driver=self.driver)
            elif not self.query:
                hook.import_query(
                    query=self.table,
                    target_dir=self.target_dir,
                    append=self.append,
                    file_type=self.file_type,
                    split_by=self.split_by,
                    direct=self.direct,
                    driver=self.driver)
            else:
                raise AirflowException(
                    "Provide query or table parameter to import using Sqoop"
                )
        else:
            raise AirflowException("cmd_type should be 'import' or 'export'")

# 需要导入模块: from airflow.contrib.hooks.sqoop_hook import SqoopHook [as 别名]
# 或者: from airflow.contrib.hooks.sqoop_hook.SqoopHook import import_query [as 别名]
    def test_submit(self):
        hook = SqoopHook(**self._config)

        cmd = ' '.join(hook._prepare_command())

        # Check if the config has been extracted from the json
        if self._config_json['namenode']:
            self.assertIn("-fs {}".format(self._config_json['namenode']), cmd)

        if self._config_json['job_tracker']:
            self.assertIn("-jt {}".format(self._config_json['job_tracker']),
                          cmd)

        if self._config_json['libjars']:
            self.assertIn("-libjars {}".format(self._config_json['libjars']),
                          cmd)

        if self._config_json['files']:
            self.assertIn("-files {}".format(self._config_json['files']), cmd)

        if self._config_json['archives']:
            self.assertIn(
                "-archives {}".format(self._config_json['archives']), cmd
            )

        self.assertIn("--hcatalog-database {}".format(self._config['hcatalog_database']), cmd)
        self.assertIn("--hcatalog-table {}".format(self._config['hcatalog_table']), cmd)

        # Check the regulator stuff passed by the default constructor
        if self._config['verbose']:
            self.assertIn("--verbose", cmd)

        if self._config['num_mappers']:
            self.assertIn(
                "--num-mappers {}".format(self._config['num_mappers']), cmd
            )

        print(self._config['properties'])
        for key, value in self._config['properties'].items():
            self.assertIn("-D {}={}".format(key, value), cmd)

        # We don't have the sqoop binary available, and this is hard to mock,
        # so just accept an exception for now.
        with self.assertRaises(OSError):
            hook.export_table(**self._config_export)

        with self.assertRaises(OSError):
            hook.import_table(table='schema.table',
                              target_dir='/sqoop/example/path')

        with self.assertRaises(OSError):
            hook.import_query(query='SELECT * FROM sometable',
                              target_dir='/sqoop/example/path')

# 需要导入模块: from airflow.contrib.hooks.sqoop_hook import SqoopHook [as 别名]
# 或者: from airflow.contrib.hooks.sqoop_hook.SqoopHook import import_query [as 别名]
    def execute(self, context):
        """
        Execute sqoop job
        """
        hook = SqoopHook(conn_id=self.conn_id,
                         verbose=self.verbose,
                         num_mappers=self.num_mappers,
                         hcatalog_database=self.hcatalog_database,
                         hcatalog_table=self.hcatalog_table,
                         properties=self.properties)

        if self.cmd_type == 'export':
            hook.export_table(
                table=self.table,
                export_dir=self.export_dir,
                input_null_string=self.input_null_string,
                input_null_non_string=self.input_null_non_string,
                staging_table=self.staging_table,
                clear_staging_table=self.clear_staging_table,
                enclosed_by=self.enclosed_by,
                escaped_by=self.escaped_by,
                input_fields_terminated_by=self.input_fields_terminated_by,
                input_lines_terminated_by=self.input_lines_terminated_by,
                input_optionally_enclosed_by=self.input_optionally_enclosed_by,
                batch=self.batch,
                relaxed_isolation=self.relaxed_isolation,
                extra_export_options=self.extra_export_options)
        elif self.cmd_type == 'import':
            # add create hcatalog table to extra import options if option passed
            # if new params are added to constructor can pass them in here so don't modify sqoop_hook for each param
            if self.create_hcatalog_table:
                self.extra_import_options['create-hcatalog-table'] = ''

            if self.table and self.query:
                raise AirflowException('Cannot specify query and table together. Need to specify either or.')

            if self.table:
                hook.import_table(
                    table=self.table,
                    target_dir=self.target_dir,
                    append=self.append,
                    file_type=self.file_type,
                    columns=self.columns,
                    split_by=self.split_by,
                    where=self.where,
                    direct=self.direct,
                    driver=self.driver,
                    extra_import_options=self.extra_import_options)
            elif self.query:
                hook.import_query(
                    query=self.query,
                    target_dir=self.target_dir,
                    append=self.append,
                    file_type=self.file_type,
                    split_by=self.split_by,
                    direct=self.direct,
                    driver=self.driver,
                    extra_import_options=self.extra_import_options)
            else:
                raise AirflowException(
                    "Provide query or table parameter to import using Sqoop"
                )
        else:
            raise AirflowException("cmd_type should be 'import' or 'export'")

# 需要导入模块: from airflow.contrib.hooks.sqoop_hook import SqoopHook [as 别名]
# 或者: from airflow.contrib.hooks.sqoop_hook.SqoopHook import import_query [as 别名]

#.........这里部分代码省略.........
        self.file_type = file_type
        self.columns = columns
        self.num_mappers = num_mappers
        self.split_by = split_by
        self.where = where
        self.export_dir = export_dir
        self.input_null_string = input_null_string
        self.input_null_non_string = input_null_non_string
        self.staging_table = staging_table
        self.clear_staging_table = clear_staging_table
        self.enclosed_by = enclosed_by
        self.escaped_by = escaped_by
        self.input_fields_terminated_by = input_fields_terminated_by
        self.input_lines_terminated_by = input_lines_terminated_by
        self.input_optionally_enclosed_by = input_optionally_enclosed_by
        self.batch = batch
        self.direct = direct
        self.driver = driver
        self.verbose = verbose
        self.relaxed_isolation = relaxed_isolation
        self.hcatalog_database = hcatalog_database
        self.hcatalog_table = hcatalog_table
        self.create_hcatalog_table = create_hcatalog_table
        self.properties = properties
        self.extra_import_options = extra_import_options or {}
        self.extra_export_options = extra_export_options or {}

    def execute(self, context):
        """
        Execute sqoop job
        """
        self.hook = SqoopHook(
            conn_id=self.conn_id,
            verbose=self.verbose,
            num_mappers=self.num_mappers,
            hcatalog_database=self.hcatalog_database,
            hcatalog_table=self.hcatalog_table,
            properties=self.properties
        )

        if self.cmd_type == 'export':
            self.hook.export_table(
                table=self.table,
                export_dir=self.export_dir,
                input_null_string=self.input_null_string,
                input_null_non_string=self.input_null_non_string,
                staging_table=self.staging_table,
                clear_staging_table=self.clear_staging_table,
                enclosed_by=self.enclosed_by,
                escaped_by=self.escaped_by,
                input_fields_terminated_by=self.input_fields_terminated_by,
                input_lines_terminated_by=self.input_lines_terminated_by,
                input_optionally_enclosed_by=self.input_optionally_enclosed_by,
                batch=self.batch,
                relaxed_isolation=self.relaxed_isolation,
                extra_export_options=self.extra_export_options)
        elif self.cmd_type == 'import':
            # add create hcatalog table to extra import options if option passed
            # if new params are added to constructor can pass them in here
            # so don't modify sqoop_hook for each param
            if self.create_hcatalog_table:
                self.extra_import_options['create-hcatalog-table'] = ''

            if self.table and self.query:
                raise AirflowException(
                    'Cannot specify query and table together. Need to specify either or.'
                )

            if self.table:
                self.hook.import_table(
                    table=self.table,
                    target_dir=self.target_dir,
                    append=self.append,
                    file_type=self.file_type,
                    columns=self.columns,
                    split_by=self.split_by,
                    where=self.where,
                    direct=self.direct,
                    driver=self.driver,
                    extra_import_options=self.extra_import_options)
            elif self.query:
                self.hook.import_query(
                    query=self.query,
                    target_dir=self.target_dir,
                    append=self.append,
                    file_type=self.file_type,
                    split_by=self.split_by,
                    direct=self.direct,
                    driver=self.driver,
                    extra_import_options=self.extra_import_options)
            else:
                raise AirflowException(
                    "Provide query or table parameter to import using Sqoop"
                )
        else:
            raise AirflowException("cmd_type should be 'import' or 'export'")

    def on_kill(self):
        self.log.info('Sending SIGTERM signal to bash process group')
        os.killpg(os.getpgid(self.hook.sp.pid), signal.SIGTERM)