本文整理汇总了Python中steppy.base.BaseTransformer方法的典型用法代码示例。如果您正苦于以下问题：Python base.BaseTransformer方法的具体用法？Python base.BaseTransformer怎么用？Python base.BaseTransformer使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类steppy.base的用法示例。
在下文中一共展示了base.BaseTransformer方法的4个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: make_apply_transformer_stream
# 需要导入模块: from steppy import base [as 别名]
# 或者: from steppy.base import BaseTransformer [as 别名]
def make_apply_transformer_stream(func, output_name='output', apply_on=None):
class StaticApplyTransformerStream(BaseTransformer):
def transform(self, *args, **kwargs):
self.check_input(*args, **kwargs)
return {output_name: self._transform(*args, **kwargs)}
def _transform(self, *args, **kwargs):
if not apply_on:
iterator = zip(*args, *kwargs.values())
else:
iterator = zip(*args, *[kwargs[key] for key in apply_on])
for func_args in tqdm(iterator):
yield func(*func_args)
@staticmethod
def check_input(*args, **kwargs):
for arg in chain(args, kwargs.values()):
if not isinstance(arg, Iterable):
raise Exception('All inputs must be iterable')
return StaticApplyTransformerStream() 示例2: make_transformer
# 需要导入模块: from steppy import base [as 别名]
# 或者: from steppy.base import BaseTransformer [as 别名]
def make_transformer(func, output_name):
class StaticTransformer(BaseTransformer):
def transform(self, *args, **kwargs):
return {output_name: func(*args, **kwargs)}
return StaticTransformer() 示例3: make_apply_transformer
# 需要导入模块: from steppy import base [as 别名]
# 或者: from steppy.base import BaseTransformer [as 别名]
def make_apply_transformer(func, output_name='output', apply_on=None):
class StaticApplyTransformer(BaseTransformer):
def transform(self, *args, **kwargs):
self.check_input(*args, **kwargs)
if not apply_on:
iterator = zip(*args, *kwargs.values())
else:
iterator = zip(*args, *[kwargs[key] for key in apply_on])
output = []
for func_args in tqdm(iterator, total=self.get_arg_length(*args, **kwargs)):
output.append(func(*func_args))
return {output_name: output}
@staticmethod
def check_input(*args, **kwargs):
if len(args) and len(kwargs) == 0:
raise Exception('Input must not be empty')
arg_length = None
for arg in chain(args, kwargs.values()):
if not isinstance(arg, Iterable):
raise Exception('All inputs must be iterable')
arg_length_loc = None
try:
arg_length_loc = len(arg)
except:
pass
if arg_length_loc is not None:
if arg_length is None:
arg_length = arg_length_loc
elif arg_length_loc != arg_length:
raise Exception('All inputs must be the same length')
@staticmethod
def get_arg_length(*args, **kwargs):
arg_length = None
for arg in chain(args, kwargs.values()):
if arg_length is None:
try:
arg_length = len(arg)
except:
pass
if arg_length is not None:
return arg_length
return StaticApplyTransformer() 示例4: make_apply_transformer
# 需要导入模块: from steppy import base [as 别名]
# 或者: from steppy.base import BaseTransformer [as 别名]
def make_apply_transformer(func, output_name='output', apply_on=None, n_threads=1):
class StaticApplyTransformer(BaseTransformer):
def transform(self, *args, **kwargs):
self.check_input(*args, **kwargs)
if not apply_on:
iterator = list(zip(*args, *kwargs.values()))
else:
iterator = list(zip(*args, *[kwargs[key] for key in apply_on]))
n_jobs = np.minimum(n_threads, len(iterator))
with mp.pool.ThreadPool(n_jobs) as executor:
output = list(tqdm(executor.imap(lambda p: func(*p), iterator), total=len(iterator)))
return {output_name: output}
@staticmethod
def check_input(*args, **kwargs):
if len(args) and len(kwargs) == 0:
raise Exception('Input must not be empty')
arg_length = None
for arg in chain(args, kwargs.values()):
if not isinstance(arg, Iterable):
raise Exception('All inputs must be iterable')
arg_length_loc = None
try:
arg_length_loc = len(arg)
except:
pass
if arg_length_loc is not None:
if arg_length is None:
arg_length = arg_length_loc
elif arg_length_loc != arg_length:
raise Exception('All inputs must be the same length')
@staticmethod
def get_arg_length(*args, **kwargs):
arg_length = None
for arg in chain(args, kwargs.values()):
if arg_length is None:
try:
arg_length = len(arg)
except:
pass
if arg_length is not None:
return arg_length
return StaticApplyTransformer()