Python base.Step方法代码示例

# 需要导入模块: from steppy import base [as 别名]
# 或者: from steppy.base import Step [as 别名]
def emptiness_preprocessing_train(config, model_name='network', suffix=''):
    reader_train = Step(name='xy_train{}'.format(suffix),
                        transformer=loaders.XYSplit(train_mode=True, **config.xy_splitter[model_name]),
                        input_data=['input'],
                        adapter=Adapter({'meta': E('input', 'meta')}),
                        experiment_directory=config.execution.experiment_dir)

    reader_inference = Step(name='xy_inference{}'.format(suffix),
                            transformer=loaders.XYSplit(train_mode=True, **config.xy_splitter[model_name]),
                            input_data=['callback_input'],
                            adapter=Adapter({'meta': E('callback_input', 'meta_valid')}),
                            experiment_directory=config.execution.experiment_dir)

    loader = Step(name='loader{}'.format(suffix),
                  transformer=loaders.EmptinessLoader(train_mode=True, **config.loaders.resize),
                  input_steps=[reader_train, reader_inference],
                  adapter=Adapter({'X': E(reader_train.name, 'X'),
                                   'y': E(reader_train.name, 'y'),
                                   'X_valid': E(reader_inference.name, 'X'),
                                   'y_valid': E(reader_inference.name, 'y'),
                                   }),
                  experiment_directory=config.execution.experiment_dir)
    return loader 

# 需要导入模块: from steppy import base [as 别名]
# 或者: from steppy.base import Step [as 别名]
def emptiness_preprocessing_inference(config, model_name='network', suffix=''):
    reader_inference = Step(name='xy_inference{}'.format(suffix),
                            transformer=loaders.XYSplit(train_mode=False, **config.xy_splitter[model_name]),
                            input_data=['input'],
                            adapter=Adapter({'meta': E('input', 'meta')}),
                            experiment_directory=config.execution.experiment_dir)

    loader = Step(name='loader{}'.format(suffix),
                  transformer=loaders.EmptinessLoader(train_mode=False, **config.loaders.resize),
                  input_steps=[reader_inference],
                  adapter=Adapter({'X': E(reader_inference.name, 'X'),
                                   'y': E(reader_inference.name, 'y'),
                                   }),
                  experiment_directory=config.execution.experiment_dir,
                  cache_output=True)
    return loader 

# 需要导入模块: from steppy import base [as 别名]
# 或者: from steppy.base import Step [as 别名]
def stacking_preprocessing_train(config, model_name='network', suffix=''):
    reader_train = Step(name='xy_train{}'.format(suffix),
                        transformer=loaders.XYSplit(train_mode=True, **config.xy_splitter[model_name]),
                        input_data=['input'],
                        adapter=Adapter({'meta': E('input', 'meta')}),
                        experiment_directory=config.execution.experiment_dir)

    reader_inference = Step(name='xy_inference{}'.format(suffix),
                            transformer=loaders.XYSplit(train_mode=True, **config.xy_splitter[model_name]),
                            input_data=['callback_input'],
                            adapter=Adapter({'meta': E('callback_input', 'meta_valid')}),
                            experiment_directory=config.execution.experiment_dir)

    loader = Step(name='loader{}'.format(suffix),
                  transformer=loaders.ImageSegmentationLoaderStacking(train_mode=True, **config.loaders.stacking),
                  input_steps=[reader_train, reader_inference],
                  adapter=Adapter({'X': E(reader_train.name, 'X'),
                                   'y': E(reader_train.name, 'y'),
                                   'X_valid': E(reader_inference.name, 'X'),
                                   'y_valid': E(reader_inference.name, 'y'),
                                   }),
                  experiment_directory=config.execution.experiment_dir)
    return loader 

# 需要导入模块: from steppy import base [as 别名]
# 或者: from steppy.base import Step [as 别名]
def stacking_preprocessing_inference(config, model_name='network', suffix=''):
    reader_inference = Step(name='xy_inference{}'.format(suffix),
                            transformer=loaders.XYSplit(train_mode=False, **config.xy_splitter[model_name]),
                            input_data=['input'],
                            adapter=Adapter({'meta': E('input', 'meta')}),
                            experiment_directory=config.execution.experiment_dir)

    loader = Step(name='loader{}'.format(suffix),
                  transformer=loaders.ImageSegmentationLoaderStacking(train_mode=False, **config.loaders.stacking),
                  input_steps=[reader_inference],
                  adapter=Adapter({'X': E(reader_inference.name, 'X'),
                                   'y': E(reader_inference.name, 'y'),
                                   }),
                  experiment_directory=config.execution.experiment_dir,
                  cache_output=True)
    return loader 

# 需要导入模块: from steppy import base [as 别名]
# 或者: from steppy.base import Step [as 别名]
def preprocessing_inference(config, model_name='network'):
    if config.general.loader_mode == 'resize':
        loader_config = config.loaders.resize
        LOADER = loaders.ImageSegmentationLoaderResize
    else:
        raise NotImplementedError

    reader_inference = Step(name='xy_inference',
                            transformer=loaders.MetaReader(train_mode=False, **config.meta_reader[model_name]),
                            input_data=['input'],
                            adapter=Adapter({'meta': E('input', 'meta')}))

    loader = Step(name='loader',
                  transformer=LOADER(train_mode=False, **loader_config),
                  input_steps=[reader_inference],
                  adapter=Adapter({'X': E(reader_inference.name, 'X'),
                                   'y': E(reader_inference.name, 'y'),
                                   }))
    return loader 

# 需要导入模块: from steppy import base [as 别名]
# 或者: from steppy.base import Step [as 别名]
def preprocessing_binary_inference(config, model_name, suffix='_binary_model'):
    reader_inference = Step(name='xy_inference{}'.format(suffix),
                            transformer=loaders.MetaReader(train_mode=True, **config.meta_reader[model_name]),
                            input_data=['input'],
                            adapter=Adapter({'meta': E('input', 'meta')}))

    transformer = OneClassImageClassificatioLoader(
        train_mode=True,
        loader_params=config.loaders.resize.loader_params,
        dataset_params=config.loaders.resize.dataset_params,
        augmentation_params=config.loaders.resize.augmentation_params
    )

    binary_loader = Step(name='loader{}'.format(suffix),
                         transformer=transformer,
                         input_steps=[reader_inference],
                         adapter=Adapter({'X': E(reader_inference.name, 'X'),
                                          }))

    return binary_loader 

# 需要导入模块: from steppy import base [as 别名]
# 或者: from steppy.base import Step [as 别名]
def data_cleaning_v2(config, train_mode, suffix, **kwargs):
    cleaned_data = data_cleaning_v1(config, train_mode, suffix, **kwargs)

    if train_mode:
        cleaned_data, cleaned_data_valid = cleaned_data

    impute_missing = Step(name='dummies_missing{}'.format(suffix),
                          transformer=dc.DummiesMissing(**config.dummies_missing),
                          input_steps=[cleaned_data],
                          adapter=Adapter({'X': E(cleaned_data.name, 'numerical_features')}),
                          experiment_directory=config.pipeline.experiment_directory, **kwargs)

    if train_mode:
        impute_missing_valid = Step(name='dummies_missing_valid{}'.format(suffix),
                                    transformer=impute_missing,
                                    input_steps=[cleaned_data_valid],
                                    adapter=Adapter({'X': E(cleaned_data_valid.name, 'numerical_features')}),
                                    experiment_directory=config.pipeline.experiment_directory, **kwargs)
        return impute_missing, impute_missing_valid
    else:
        return impute_missing 

# 需要导入模块: from steppy import base [as 别名]
# 或者: from steppy.base import Step [as 别名]
def row_aggregation_features(config, train_mode, suffix, **kwargs):
    bucket_nrs = config.row_aggregations.bucket_nrs
    row_agg_features = []
    for bucket_nr in bucket_nrs:
        row_agg_feature = Step(name='row_agg_feature_bucket_nr{}{}'.format(bucket_nr, suffix),
                               transformer=fe.RowAggregationFeatures(bucket_nr=bucket_nr),
                               input_data=['input'],
                               adapter=Adapter({'X': E('input', 'X')}),
                               experiment_directory=config.pipeline.experiment_directory, **kwargs)
        row_agg_features.append(row_agg_feature)

    if train_mode:
        row_agg_features_valid = []
        for bucket_nr, row_agg_feature in zip(bucket_nrs, row_agg_features):
            row_agg_feature_valid = Step(name='row_agg_feature_bucket_nr{}_valid{}'.format(bucket_nr, suffix),
                                         transformer=row_agg_feature,
                                         input_data=['input'],
                                         adapter=Adapter({'X': E('input', 'X_valid')}),
                                         experiment_directory=config.pipeline.experiment_directory, **kwargs)
            row_agg_features_valid.append(row_agg_feature_valid)

        return row_agg_features, row_agg_features_valid
    else:
        return row_agg_features 

# 需要导入模块: from steppy import base [as 别名]
# 或者: from steppy.base import Step [as 别名]
def classifier_catboost(features, config, train_mode, suffix, **kwargs):
    model_name = 'catboost{}'.format(suffix)

    if train_mode:
        features_train, features_valid = features

        catboost = Step(name=model_name,
                        transformer=CatBoost(**config.catboost),
                        input_data=['main_table'],
                        input_steps=[features_train, features_valid],
                        adapter=Adapter({'X': E(features_train.name, 'features'),
                                         'y': E('main_table', 'y'),
                                         'feature_names': E(features_train.name, 'feature_names'),
                                         'categorical_features': E(features_train.name, 'categorical_features'),
                                         'X_valid': E(features_valid.name, 'features'),
                                         'y_valid': E('main_table', 'y_valid'),
                                         }),
                        experiment_directory=config.pipeline.experiment_directory, **kwargs)
    else:
        catboost = Step(name=model_name,
                        transformer=CatBoost(**config.catboost),
                        input_steps=[features],
                        adapter=Adapter({'X': E(features.name, 'features')}),
                        experiment_directory=config.pipeline.experiment_directory, **kwargs)
    return catboost 

# 需要导入模块: from steppy import base [as 别名]
# 或者: from steppy.base import Step [as 别名]
def classifier_xgb(features, config, train_mode, suffix, **kwargs):
    if train_mode:
        features_train, features_valid = features

        xgboost = Step(name='xgboost{}'.format(suffix),
                       transformer=XGBoost(**config.xgboost),
                       input_data=['main_table'],
                       input_steps=[features_train, features_valid],
                       adapter=Adapter({'X': E(features_train.name, 'features'),
                                        'y': E('main_table', 'y'),
                                        'feature_names': E(features_train.name, 'feature_names'),
                                        'X_valid': E(features_valid.name, 'features'),
                                        'y_valid': E('main_table', 'y_valid'),
                                        }),
                       experiment_directory=config.pipeline.experiment_directory,
                       **kwargs)
    else:
        xgboost = Step(name='xgboost{}'.format(suffix),
                       transformer=XGBoost(**config.xgboost),
                       input_steps=[features],
                       adapter=Adapter({'X': E(features.name, 'features')}),
                       experiment_directory=config.pipeline.experiment_directory,
                       **kwargs)
    return xgboost 

# 需要导入模块: from steppy import base [as 别名]
# 或者: from steppy.base import Step [as 别名]
def visualizer(model, label_encoder, config):
    label_decoder = Step(name='label_decoder',
                         transformer=GoogleAiLabelDecoder(),
                         input_steps=[label_encoder, ],
                         experiment_directory=config.env.cache_dirpath)

    decoder = Step(name='decoder',
                   transformer=DataDecoder(**config.postprocessing.data_decoder),
                   input_data=['input'],
                   input_steps=[model, ],
                   experiment_directory=config.env.cache_dirpath)

    visualize = Step(name='visualizer',
                     transformer=Visualizer(),
                     input_steps=[label_decoder, decoder],
                     input_data=['input'],
                     adapter=Adapter({'images_data': E('input', 'images_data'),
                                      'results': E(decoder.name, 'results'),
                                      'decoder_dict': E(label_decoder.name, 'inverse_mapping')}),
                     experiment_directory=config.env.cache_dirpath)

    return visualize 

# 需要导入模块: from steppy import base [as 别名]
# 或者: from steppy.base import Step [as 别名]
def postprocessing(model, label_encoder, config):
    label_decoder = Step(name='label_decoder',
                         transformer=GoogleAiLabelDecoder(),
                         input_steps=[label_encoder, ],
                         experiment_directory=config.env.cache_dirpath)

    decoder = Step(name='decoder',
                   transformer=DataDecoder(**config.postprocessing.data_decoder),
                   input_data=['input'],
                   input_steps=[model, ],
                   experiment_directory=config.env.cache_dirpath)

    submission_producer = Step(name='submission_producer',
                               transformer=PredictionFormatter(),
                               input_steps=[label_decoder, decoder],
                               input_data=['input'],
                               adapter=Adapter({'images_data': E('input', 'images_data'),
                                                'results': E(decoder.name, 'results'),
                                                'decoder_dict': E(label_decoder.name, 'inverse_mapping')}),
                               experiment_directory=config.env.cache_dirpath)
    return submission_producer 

# 需要导入模块: from steppy import base [as 别名]
# 或者: from steppy.base import Step [as 别名]
def test_inputs_without_conflicting_names_do_not_require_adapter(data):
    step = Step(
        name='test_inputs_without_conflicting_names_do_not_require_adapter_1',
        transformer=IdentityOperation(),
        input_data=['input_1']
    )
    output = step.fit_transform(data)
    assert output == data['input_1']

    step = Step(
        name='test_inputs_without_conflicting_names_do_not_require_adapter_2',
        transformer=IdentityOperation(),
        input_data=['input_1', 'input_2']
    )
    output = step.fit_transform(data)
    assert output == {**data['input_1'], **data['input_2']} 

# 需要导入模块: from steppy import base [as 别名]
# 或者: from steppy.base import Step [as 别名]
def test_step_with_adapted_inputs(data):
    step = Step(
        name='test_step_wit_adapted_inputs',
        transformer=IdentityOperation(),
        input_data=['input_1', 'input_3'],
        adapter=Adapter({
            'img': E('input_3', 'images'),
            'fea': E('input_1', 'features'),
            'l1': E('input_3', 'labels'),
            'l2': E('input_1', 'labels'),
        })
    )
    output = step.fit_transform(data)
    expected = {
        'img': data['input_3']['images'],
        'fea': data['input_1']['features'],
        'l1': data['input_3']['labels'],
        'l2': data['input_1']['labels'],
    }
    assert output == expected 

# 需要导入模块: from steppy import base [as 别名]
# 或者: from steppy.base import Step [as 别名]
def network(config, suffix='', train_mode=True):
    if train_mode:
        preprocessing = emptiness_preprocessing_train(config, model_name='network', suffix=suffix)
    else:
        preprocessing = emptiness_preprocessing_inference(config, suffix=suffix)

    network = utils.FineTuneStep(name='network{}'.format(suffix),
                                 transformer=models.SegmentationModel(**config.model['network']),
                                 input_data=['callback_input'],
                                 input_steps=[preprocessing],
                                 adapter=Adapter({'datagen': E(preprocessing.name, 'datagen'),
                                                  'validation_datagen': E(preprocessing.name, 'validation_datagen'),
                                                  'meta_valid': E('callback_input', 'meta_valid'),
                                                  }),
                                 is_trainable=True,
                                 fine_tuning=config.model.network.training_config.fine_tuning,
                                 experiment_directory=config.execution.experiment_dir)

    mask_resize = Step(name='mask_resize{}'.format(suffix),
                       transformer=utils.make_apply_transformer(partial(postprocessing.resize_emptiness_predictions,
                                                                        target_size=config.general.original_size),
                                                                output_name='resized_images',
                                                                apply_on=['images']),
                       input_steps=[network],
                       adapter=Adapter({'images': E(network.name, 'mask_prediction'),
                                        }),
                       experiment_directory=config.execution.experiment_dir)

    return mask_resize


#   __________   ___  _______   ______  __    __  .___________. __    ______   .__   __.
#  |   ____\  \ /  / |   ____| /      ||  |  |  | |           ||  |  /  __  \  |  \ |  |
#  |  |__   \  V  /  |  |__   |  ,----'|  |  |  | `---|  |----`|  | |  |  |  | |   \|  |
#  |   __|   >   <   |   __|  |  |     |  |  |  |     |  |     |  | |  |  |  | |  . `  |
#  |  |____ /  .  \  |  |____ |  `----.|  `--'  |     |  |     |  | |  `--'  | |  |\   |
#  |_______/__/ \__\ |_______| \______| \______/      |__|     |__|  \______/  |__| \__|
#