本文整理汇总了Python中apache_beam.runners.DataflowRunner.flatten_input_visitor方法的典型用法代码示例。如果您正苦于以下问题：Python DataflowRunner.flatten_input_visitor方法的具体用法？Python DataflowRunner.flatten_input_visitor怎么用？Python DataflowRunner.flatten_input_visitor使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类apache_beam.runners.DataflowRunner的用法示例。
在下文中一共展示了DataflowRunner.flatten_input_visitor方法的2个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: _test_flatten_input_visitor
# 需要导入模块: from apache_beam.runners import DataflowRunner [as 别名]
# 或者: from apache_beam.runners.DataflowRunner import flatten_input_visitor [as 别名]
def _test_flatten_input_visitor(self, input_type, output_type, num_inputs):
p = TestPipeline()
inputs = []
for _ in range(num_inputs):
input_pcoll = PCollection(p)
input_pcoll.element_type = input_type
inputs.append(input_pcoll)
output_pcoll = PCollection(p)
output_pcoll.element_type = output_type
flatten = AppliedPTransform(None, beam.Flatten(), "label", inputs)
flatten.add_output(output_pcoll, None)
DataflowRunner.flatten_input_visitor().visit_transform(flatten)
for _ in range(num_inputs):
self.assertEqual(inputs[0].element_type, output_type)示例2: test_gbk_then_flatten_input_visitor
# 需要导入模块: from apache_beam.runners import DataflowRunner [as 别名]
# 或者: from apache_beam.runners.DataflowRunner import flatten_input_visitor [as 别名]
def test_gbk_then_flatten_input_visitor(self):
p = TestPipeline(
runner=DataflowRunner(),
options=PipelineOptions(self.default_properties))
none_str_pc = p | 'c1' >> beam.Create({None: 'a'})
none_int_pc = p | 'c2' >> beam.Create({None: 3})
flat = (none_str_pc, none_int_pc) | beam.Flatten()
_ = flat | beam.GroupByKey()
# This may change if type inference changes, but we assert it here
# to make sure the check below is not vacuous.
self.assertNotIsInstance(flat.element_type, typehints.TupleConstraint)
p.visit(DataflowRunner.group_by_key_input_visitor())
p.visit(DataflowRunner.flatten_input_visitor())
# The dataflow runner requires gbk input to be tuples *and* flatten
# inputs to be equal to their outputs. Assert both hold.
self.assertIsInstance(flat.element_type, typehints.TupleConstraint)
self.assertEqual(flat.element_type, none_str_pc.element_type)
self.assertEqual(flat.element_type, none_int_pc.element_type)