Python base.IdentityOperation方法代碼示例

# 需要導入模塊: from steppy import base [as 別名]
# 或者: from steppy.base import IdentityOperation [as 別名]
def test_inputs_without_conflicting_names_do_not_require_adapter(data):
    step = Step(
        name='test_inputs_without_conflicting_names_do_not_require_adapter_1',
        transformer=IdentityOperation(),
        input_data=['input_1']
    )
    output = step.fit_transform(data)
    assert output == data['input_1']

    step = Step(
        name='test_inputs_without_conflicting_names_do_not_require_adapter_2',
        transformer=IdentityOperation(),
        input_data=['input_1', 'input_2']
    )
    output = step.fit_transform(data)
    assert output == {**data['input_1'], **data['input_2']} 

# 需要導入模塊: from steppy import base [as 別名]
# 或者: from steppy.base import IdentityOperation [as 別名]
def test_step_with_adapted_inputs(data):
    step = Step(
        name='test_step_wit_adapted_inputs',
        transformer=IdentityOperation(),
        input_data=['input_1', 'input_3'],
        adapter=Adapter({
            'img': E('input_3', 'images'),
            'fea': E('input_1', 'features'),
            'l1': E('input_3', 'labels'),
            'l2': E('input_1', 'labels'),
        })
    )
    output = step.fit_transform(data)
    expected = {
        'img': data['input_3']['images'],
        'fea': data['input_1']['features'],
        'l1': data['input_3']['labels'],
        'l2': data['input_1']['labels'],
    }
    assert output == expected 

# 需要導入模塊: from steppy import base [as 別名]
# 或者: from steppy.base import IdentityOperation [as 別名]
def postprocessing_pipeline_simplified(cache_dirpath, loader_mode, threshold):
    if loader_mode == 'resize_and_pad':
        size_adjustment_function = partial(crop_image, target_size=ORIGINAL_SIZE)
    elif loader_mode == 'resize' or loader_mode == 'stacking':
        size_adjustment_function = partial(resize_image, target_size=ORIGINAL_SIZE)
    else:
        raise NotImplementedError

    mask_resize = Step(name='mask_resize',
                       transformer=make_apply_transformer(size_adjustment_function,
                                                          output_name='resized_images',
                                                          apply_on=['images']),
                       input_data=['network_output'],
                       adapter=Adapter({'images': E('network_output', 'mask_prediction'),
                                        }),
                       experiment_directory=cache_dirpath)

    binarizer = Step(name='binarizer',
                     transformer=make_apply_transformer(
                         partial(binarize, threshold=threshold),
                         output_name='binarized_images',
                         apply_on=['images']),
                     input_steps=[mask_resize],
                     adapter=Adapter({'images': E(mask_resize.name, 'resized_images'),
                                      }),
                     experiment_directory=cache_dirpath)

    output = Step(name='output',
                  transformer=IdentityOperation(),
                  input_steps=[binarizer],
                  adapter=Adapter({'y_pred': E(binarizer.name, 'binarized_images'),
                                   }),
                  experiment_directory=cache_dirpath)

    return output 

# 需要導入模塊: from steppy import base [as 別名]
# 或者: from steppy.base import IdentityOperation [as 別名]
def oof_predictions(config, train_mode, suffix, **kwargs):
    features = Step(name='oof_predictions{}'.format(suffix),
                    transformer=IdentityOperation(),
                    input_data=['oof_predictions'],
                    adapter=Adapter({'numerical_features': E('oof_predictions', 'X')
                                     }),
                    experiment_directory=config.pipeline.experiment_directory, **kwargs)

    feature_combiner = _join_features(numerical_features=[features],
                                      categorical_features=[],
                                      config=config, train_mode=train_mode, suffix=suffix, **kwargs)
    if train_mode:
        features_valid = Step(name='oof_predictions{}'.format(suffix),
                              transformer=IdentityOperation(),
                              input_data=['oof_predictions'],
                              adapter=Adapter({'numerical_features': E('oof_predictions', 'X_valid')
                                               }),
                              experiment_directory=config.pipeline.experiment_directory, **kwargs)

        feature_combiner_valid = _join_features(numerical_features=[features_valid],
                                                categorical_features=[],
                                                config=config, train_mode=train_mode, suffix='_valid{}'.format(suffix),
                                                **kwargs)
        return feature_combiner, feature_combiner_valid
    else:
        return feature_combiner 

# 需要導入模塊: from steppy import base [as 別名]
# 或者: from steppy.base import IdentityOperation [as 別名]
def retinanet(config, train_mode, visualize=False):
    persist_output = False
    load_persisted_output = False

    loader = preprocessing_generator(config, is_train=train_mode)

    retinanet = Step(name='retinanet',
                     transformer=Retina(**config.retinanet, train_mode=train_mode),
                     input_steps=[loader],
                     experiment_directory=config.env.cache_dirpath,
                     persist_output=persist_output,
                     is_trainable=True,
                     load_persisted_output=load_persisted_output)

    if train_mode:
        return retinanet

    if visualize:
        return visualizer(retinanet, loader.get_step('label_encoder'), config)

    postprocessor = postprocessing(retinanet, loader.get_step('label_encoder'), config)

    output = Step(name='output',
                  transformer=IdentityOperation(),
                  input_steps=[postprocessor],
                  adapter=Adapter({'y_pred': E(postprocessor.name, 'submission')}),
                  experiment_directory=config.env.cache_dirpath,
                  persist_output=persist_output,
                  load_persisted_output=load_persisted_output)
    return output 

# 需要導入模塊: from steppy import base [as 別名]
# 或者: from steppy.base import IdentityOperation [as 別名]
def test_inputs_with_conflicting_names_require_adapter(data):
    step = Step(
        name='test_inputs_with_conflicting_names_require_adapter',
        transformer=IdentityOperation(),
        input_data=['input_1', 'input_3']
    )
    with pytest.raises(StepError):
        step.fit_transform(data) 

# 需要導入模塊: from steppy import base [as 別名]
# 或者: from steppy.base import IdentityOperation [as 別名]
def network_tta(config, suffix=''):
    if SECOND_LEVEL:
        raise NotImplementedError('Second level does not work with TTA')

    preprocessing, tta_generator = pipelines.preprocessing_inference_tta(config, model_name='network')

    if USE_DEPTH:
        Network = models.SegmentationModelWithDepth
    else:
        Network = models.SegmentationModel

    network = Step(name='network{}'.format(suffix),
                   transformer=Network(**config.model['network']),
                   input_data=['callback_input'],
                   input_steps=[preprocessing],
                   is_trainable=True,
                   experiment_directory=config.execution.experiment_dir)

    tta_aggregator = pipelines.aggregator('tta_aggregator{}'.format(suffix), network,
                                          tta_generator=tta_generator,
                                          experiment_directory=config.execution.experiment_dir,
                                          config=config.tta_aggregator)

    prediction_renamed = Step(name='prediction_renamed{}'.format(suffix),
                              transformer=IdentityOperation(),
                              input_steps=[tta_aggregator],
                              adapter=Adapter({'mask_prediction': E(tta_aggregator.name, 'aggregated_prediction')
                                               }),
                              experiment_directory=config.execution.experiment_dir)

    if config.general.loader_mode == 'resize_and_pad':
        size_adjustment_function = partial(postprocessing.crop_image, target_size=config.general.original_size)
    elif config.general.loader_mode == 'resize' or config.general.loader_mode == 'stacking':
        size_adjustment_function = partial(postprocessing.resize_image, target_size=config.general.original_size)
    else:
        raise NotImplementedError

    mask_resize = Step(name='mask_resize{}'.format(suffix),
                       transformer=utils.make_apply_transformer(size_adjustment_function,
                                                                output_name='resized_images',
                                                                apply_on=['images']),
                       input_steps=[prediction_renamed],
                       adapter=Adapter({'images': E(prediction_renamed.name, 'mask_prediction'),
                                        }),
                       experiment_directory=config.execution.experiment_dir)

    return mask_resize


#   __________   ___  _______   ______  __    __  .___________. __    ______   .__   __.
#  |   ____\  \ /  / |   ____| /      ||  |  |  | |           ||  |  /  __  \  |  \ |  |
#  |  |__   \  V  /  |  |__   |  ,----'|  |  |  | `---|  |----`|  | |  |  |  | |   \|  |
#  |   __|   >   <   |   __|  |  |     |  |  |  |     |  |     |  | |  |  |  | |  . `  |
#  |  |____ /  .  \  |  |____ |  `----.|  `--'  |     |  |     |  | |  `--'  | |  |\   |
#  |_______/__/ \__\ |_______| \______| \______/      |__|     |__|  \______/  |__| \__|
# 

# 需要導入模塊: from steppy import base [as 別名]
# 或者: from steppy.base import IdentityOperation [as 別名]
def catboost_preprocessing(features, config, train_mode, suffix, **kwargs):
    if train_mode:
        features, features_valid = features

    fillnaer = Step(name='fillna{}'.format(suffix),
                    transformer=_fillna(**config.sklearn_preprocessing.fillna),
                    input_steps=[features],
                    adapter=Adapter({'X': E(features.name, 'features')}),
                    experiment_directory=config.pipeline.experiment_directory,
                    )

    preprocessed = Step(name='preprocess{}'.format(suffix),
                        transformer=IdentityOperation(),
                        input_steps=[fillnaer, features],
                        adapter=Adapter({'features': E(fillnaer.name, 'transformed'),
                                         'feature_names': E(features.name, 'feature_names'),
                                         'categorical_features': E(features.name, 'categorical_features')
                                         }),
                        experiment_directory=config.pipeline.experiment_directory,
                        **kwargs
                        )

    if train_mode:
        fillnaer_valid = Step(name='fillna_valid{}'.format(suffix),
                              transformer=fillnaer,
                              input_steps=[features_valid],
                              adapter=Adapter({'X': E(features_valid.name, 'features')}),
                              experiment_directory=config.pipeline.experiment_directory,
                              )
        preprocessed_valid = Step(name='preprocess_valid{}'.format(suffix),
                                  transformer=IdentityOperation(),
                                  input_steps=[fillnaer_valid, features_valid],
                                  adapter=Adapter({'features': E(fillnaer_valid.name, 'transformed'),
                                                   'feature_names': E(features_valid.name,
                                                                      'feature_names'),
                                                   'categorical_features': E(features_valid.name,
                                                                             'categorical_features')
                                                   }),
                                  experiment_directory=config.pipeline.experiment_directory,
                                  **kwargs
                                  )
        return preprocessed, preprocessed_valid
    else:
        return preprocessed 

# 需要導入模塊: from steppy import base [as 別名]
# 或者: from steppy.base import IdentityOperation [as 別名]
def stacking_normalization(features, config, train_mode, suffix, **kwargs):
    if train_mode:
        features, features_valid = features

    normalizer = Step(name='stacking_normalizer{}'.format(suffix),
                      transformer=Normalizer(),
                      input_steps=[features],
                      adapter=Adapter({'X': E(features.name, 'features')}),
                      experiment_directory=config.pipeline.experiment_directory,
                      )

    stacking_normalized = Step(name='stacking_normalization{}'.format(suffix),
                               transformer=IdentityOperation(),
                               input_steps=[normalizer, features],
                               adapter=Adapter({'features': E(normalizer.name, 'X'),
                                                'feature_names': E(features.name, 'feature_names'),
                                                'categorical_features': E(features.name, 'categorical_features')
                                                }),
                               experiment_directory=config.pipeline.experiment_directory,
                               **kwargs
                               )

    if train_mode:
        normalizer_valid = Step(name='stacking_normalizer_valid{}'.format(suffix),
                                transformer=normalizer,
                                input_steps=[features_valid],
                                adapter=Adapter({'X': E(features_valid.name, 'features')}),
                                experiment_directory=config.pipeline.experiment_directory,
                                )

        stacking_normalized_valid = Step(name='stacking_normalization_valid{}'.format(suffix),
                                         transformer=IdentityOperation(),
                                         input_steps=[normalizer_valid, features_valid],
                                         adapter=Adapter({'features': E(normalizer_valid.name, 'X'),
                                                          'feature_names': E(features_valid.name, 'feature_names'),
                                                          'categorical_features': E(features_valid.name,
                                                                                    'categorical_features')
                                                          }),
                                         experiment_directory=config.pipeline.experiment_directory,
                                         **kwargs
                                         )
        return stacking_normalized, stacking_normalized_valid
    else:
        return stacking_normalized 

# 需要導入模塊: from steppy import base [as 別名]
# 或者: from steppy.base import IdentityOperation [as 別名]
def postprocessing_pipeline_simplified(cache_dirpath):
    mask_resize = Step(name='mask_resize',
                       transformer=make_apply_transformer(post.resize_image,
                                                          output_name='resized_images',
                                                          apply_on=['images', 'target_sizes']),
                       input_data=['unet_output', 'callback_input'],
                       adapter={'images': ([('unet_output', 'multichannel_map_prediction')]),
                                'target_sizes': ([('callback_input', 'target_sizes')]),
                                },
                       cache_dirpath=cache_dirpath)

    category_mapper = Step(name='category_mapper',
                           transformer=make_apply_transformer(post.categorize_image,
                                                              output_name='categorized_images'),
                           input_steps=[mask_resize],
                           adapter={'images': ([('mask_resize', 'resized_images')]),
                                    },
                           cache_dirpath=cache_dirpath)

    labeler = Step(name='labeler',
                   transformer=make_apply_transformer(post.label_multiclass_image,
                                                      output_name='labeled_images'),
                   input_steps=[category_mapper],
                   adapter={'images': ([(category_mapper.name, 'categorized_images')]),
                            },
                   cache_dirpath=cache_dirpath)

    score_builder = Step(name='score_builder',
                         transformer=make_apply_transformer(post.build_score,
                                                            output_name='images_with_scores',
                                                            apply_on=['images', 'probabilities']),
                         input_steps=[labeler, mask_resize],
                         adapter={'images': ([(labeler.name, 'labeled_images')]),
                                  'probabilities': ([(mask_resize.name, 'resized_images')]),
                                  },
                         cache_dirpath=cache_dirpath)

    output = Step(name='output',
                  transformer=IdentityOperation(),
                  input_steps=[score_builder],
                  adapter={'y_pred': ([(score_builder.name, 'images_with_scores')]),
                           },
                  cache_dirpath=cache_dirpath)

    return output