Python python_operator.PythonOperator方法代碼示例

# 需要導入模塊: from airflow.operators import python_operator [as 別名]
# 或者: from airflow.operators.python_operator import PythonOperator [as 別名]
def get_dated_main_runner_operator(
        dag,
        main_function,
        execution_timeout,
        day_shift=0,
        task_id='pull_image_data',
):
    args_str = f'{{{{ macros.ds_add(ds, -{day_shift}) }}}}'
    return PythonOperator(
        task_id=task_id,
        python_callable=main_function,
        op_args=[args_str],
        execution_timeout=execution_timeout,
        depends_on_past=False,
        dag=dag
    ) 

# 需要導入模塊: from airflow.operators import python_operator [as 別名]
# 或者: from airflow.operators.python_operator import PythonOperator [as 別名]
def get_action_xcom(self, task_id=dn.ACTION_XCOM):
        """Generate the action_xcom step

        Step responsible for getting the action information passed
        by the invocation of the dag, which includes any options.
        """
        def xcom_push(**kwargs):
            """xcom_push function

            Defines a push function to store the content of 'action' that is
            defined via 'dag_run' in XCOM so that it can be used by the
            Operators. Includes action-related information for later steps.
            """

            kwargs['ti'].xcom_push(key='action',
                                   value=kwargs['dag_run'].conf['action'])
            kwargs['ti'].xcom_push(key='action_type',
                                   value=self.action_type)

        return PythonOperator(task_id=task_id,
                              dag=self.dag,
                              python_callable=xcom_push) 

# 需要導入模塊: from airflow.operators import python_operator [as 別名]
# 或者: from airflow.operators.python_operator import PythonOperator [as 別名]
def generate_parallel_tasks(name_prefix, num_of_tasks, deps):
    """
    Generate a list of PythonOperator tasks. The generated tasks are set up to
    be dependent on the `deps` argument.
    """
    tasks = []
    for t_id in range(num_of_tasks):
        run_this = PythonOperator(
            task_id=f"{name_prefix}_{t_id}",
            python_callable=print_context,
        )
        run_this << deps
        tasks.append(run_this)
    return tasks 

# 需要導入模塊: from airflow.operators import python_operator [as 別名]
# 或者: from airflow.operators.python_operator import PythonOperator [as 別名]
def get_runner_operator(dag):
    return PythonOperator(
        task_id='pull_wikimedia_commons_data',
        python_callable=wikimedia_commons.main,
        op_args=['{{ ds }}'],
        depends_on_past=False,
        dag=dag
    ) 

# 需要導入模塊: from airflow.operators import python_operator [as 別名]
# 或者: from airflow.operators.python_operator import PythonOperator [as 別名]
def get_runner_operator(dag):
    return PythonOperator(
        task_id="pull_rawpixel_data",
        python_callable=raw_pixel.main,
        depends_on_past=False,
        dag=dag
    ) 

# 需要導入模塊: from airflow.operators import python_operator [as 別名]
# 或者: from airflow.operators.python_operator import PythonOperator [as 別名]
def get_runner_operator(dag):
    return PythonOperator(
        task_id='pull_metropolitan_museum_data',
        python_callable=metropolitan_museum_of_art.main,
        op_args=['{{ ds }}'],
        depends_on_past=False,
        dag=dag
    ) 

# 需要導入模塊: from airflow.operators import python_operator [as 別名]
# 或者: from airflow.operators.python_operator import PythonOperator [as 別名]
def get_runner_operator(dag):
    return PythonOperator(
        task_id='pull_europeana_data',
        python_callable=europeana.main,
        op_args=['{{ ds }}'],
        depends_on_past=False,
        dag=dag
    ) 

# 需要導入模塊: from airflow.operators import python_operator [as 別名]
# 或者: from airflow.operators.python_operator import PythonOperator [as 別名]
def get_runner_operator(dag):
    return PythonOperator(
        task_id="pull_cleveland_data",
        python_callable=cleveland_museum_of_art.main,
        depends_on_past=False,
        dag=dag
    ) 

# 需要導入模塊: from airflow.operators import python_operator [as 別名]
# 或者: from airflow.operators.python_operator import PythonOperator [as 別名]
def get_table_creator_operator(
        dag,
        postgres_conn_id,
        identifier=TIMESTAMP_TEMPLATE
):
    return PythonOperator(
        task_id='create_loading_table',
        python_callable=sql.create_loading_table,
        op_args=[postgres_conn_id, identifier],
        dag=dag
    ) 

# 需要導入模塊: from airflow.operators import python_operator [as 別名]
# 或者: from airflow.operators.python_operator import PythonOperator [as 別名]
def get_load_local_data_operator(
        dag,
        output_dir,
        postgres_conn_id,
        identifier=TIMESTAMP_TEMPLATE
):
    return PythonOperator(
        task_id='load_local_data',
        python_callable=loader.load_local_data,
        op_args=[output_dir, postgres_conn_id, identifier],
        trigger_rule=TriggerRule.ALL_SUCCESS,
        dag=dag
    ) 

# 需要導入模塊: from airflow.operators import python_operator [as 別名]
# 或者: from airflow.operators.python_operator import PythonOperator [as 別名]
def get_copy_to_s3_operator(
        dag,
        output_dir,
        storage_bucket,
        aws_conn_id,
        identifier=TIMESTAMP_TEMPLATE
):
    return PythonOperator(
        task_id='copy_to_s3',
        python_callable=loader.copy_to_s3,
        op_args=[output_dir, storage_bucket, identifier, aws_conn_id],
        dag=dag
    ) 

# 需要導入模塊: from airflow.operators import python_operator [as 別名]
# 或者: from airflow.operators.python_operator import PythonOperator [as 別名]
def get_load_s3_data_operator(
        dag,
        bucket,
        aws_conn_id,
        postgres_conn_id,
        identifier=TIMESTAMP_TEMPLATE
):
    return PythonOperator(
        task_id='load_s3_data',
        python_callable=loader.load_s3_data,
        op_args=[bucket, aws_conn_id, postgres_conn_id, identifier],
        dag=dag
    ) 

# 需要導入模塊: from airflow.operators import python_operator [as 別名]
# 或者: from airflow.operators.python_operator import PythonOperator [as 別名]
def get_file_deletion_operator(
        dag,
        output_dir,
        identifier=TIMESTAMP_TEMPLATE
):
    return PythonOperator(
        task_id='delete_staged_file',
        python_callable=paths.delete_staged_file,
        op_args=[output_dir, identifier],
        trigger_rule=TriggerRule.ALL_SUCCESS,
        dag=dag,
    ) 

# 需要導入模塊: from airflow.operators import python_operator [as 別名]
# 或者: from airflow.operators.python_operator import PythonOperator [as 別名]
def get_failure_moving_operator(
        dag,
        output_dir,
        identifier=TIMESTAMP_TEMPLATE
):
    return PythonOperator(
        task_id='move_staged_failures',
        python_callable=paths.move_staged_files_to_failure_directory,
        op_args=[output_dir, identifier],
        trigger_rule=TriggerRule.ONE_SUCCESS,
        dag=dag
    ) 

# 需要導入模塊: from airflow.operators import python_operator [as 別名]
# 或者: from airflow.operators.python_operator import PythonOperator [as 別名]
def get_flickr_sub_provider_update_operator(
        dag,
        postgres_conn_id,
):
    return PythonOperator(
        task_id='update_flickr_sub_providers',
        python_callable=sql.update_flickr_sub_providers,
        op_args=[postgres_conn_id],
        dag=dag
    )