Python dsl.ContainerOp方法代码示例

# 需要导入模块: from kfp import dsl [as 别名]
# 或者: from kfp.dsl import ContainerOp [as 别名]
def test_init_container(self):
    echo = dsl.UserContainer(
      name='echo',
      image='alpine:latest',
      command=['echo', 'bye'])

    @dsl.pipeline(name='InitContainer', description='A pipeline with init container.')
    def init_container_pipeline():
      dsl.ContainerOp(
        name='hello',
        image='alpine:latest',
        command=['echo', 'hello'],
        init_containers=[echo])

    workflow_dict = compiler.Compiler()._compile(init_container_pipeline)
    for template in workflow_dict['spec']['templates']:
      init_containers = template.get('initContainers', None)
      if init_containers:
        self.assertEqual(len(init_containers),1)
        init_container = init_containers[0]
        self.assertEqual(init_container, {'image':'alpine:latest', 'command': ['echo', 'bye'], 'name': 'echo'}) 

# 需要导入模块: from kfp import dsl [as 别名]
# 或者: from kfp.dsl import ContainerOp [as 别名]
def pipeline(my_pipe_param=10):
    loop_args = [{'A_a': 1, 'B_b': 2}, {'A_a': 10, 'B_b': 20}]
    with dsl.ParallelFor(loop_args) as item:
        op1 = dsl.ContainerOp(
            name="my-in-coop1",
            image="library/bash:4.4.23",
            command=["sh", "-c"],
            arguments=["echo op1 %s %s" % (item.A_a, my_pipe_param)],
        )

        op2 = dsl.ContainerOp(
            name="my-in-coop2",
            image="library/bash:4.4.23",
            command=["sh", "-c"],
            arguments=["echo op2 %s" % item.B_b],
        )

    op_out = dsl.ContainerOp(
        name="my-out-cop",
        image="library/bash:4.4.23",
        command=["sh", "-c"],
        arguments=["echo %s" % my_pipe_param],
    ) 

# 需要导入模块: from kfp import dsl [as 别名]
# 或者: from kfp.dsl import ContainerOp [as 别名]
def _proxy_container_op_props(cls: "ContainerOp"):
    """Takes the `ContainerOp` class and proxy the PendingDeprecation properties
    in `ContainerOp` to the `Container` instance.
    """
    # properties mapping to proxy: ContainerOps.<prop> => Container.<prop>
    prop_map = dict(image='image', env_variables='env')
    # itera and create class props
    for op_prop, container_prop in prop_map.items():
        # create getter and setter
        _getter, _setter = _create_getter_setter(container_prop)
        # decorate with deprecation warning
        getter = deprecation_warning(_getter, op_prop, container_prop)
        setter = deprecation_warning(_setter, op_prop, container_prop)
        # update attribites with properties
        setattr(cls, op_prop, property(getter, setter))
    return cls 

# 需要导入模块: from kfp import dsl [as 别名]
# 或者: from kfp.dsl import ContainerOp [as 别名]
def container(self):
        """`Container` object that represents the `container` property in 
        `io.argoproj.workflow.v1alpha1.Template`. Can be used to update the
        container configurations. 
        
        Example:
            import kfp.dsl as dsl
            from kubernetes.client.models import V1EnvVar
    
            @dsl.pipeline(name='example_pipeline')
            def immediate_value_pipeline():
                op1 = (dsl.ContainerOp(name='example', image='nginx:alpine')
                          .container
                            .add_env_variable(V1EnvVar(name='HOST', value='foo.bar'))
                            .add_env_variable(V1EnvVar(name='PORT', value='80'))
                            .parent # return the parent `ContainerOp`
                        )
        """
        return self._container 

# 需要导入模块: from kfp import dsl [as 别名]
# 或者: from kfp.dsl import ContainerOp [as 别名]
def test_basic(self):
        """Test basic usage."""
        def my_pipeline():
            vol = VolumeOp(
                name="myvol_creation",
                resource_name="myvol",
                size="1Gi"
            )
            op1 = ContainerOp(
                name="op1",
                image="image",
                pvolumes={"/mnt": vol.volume}
            )
            op2 = ContainerOp(
                name="op2",
                image="image",
                pvolumes={"/data": op1.pvolume}
            )

            self.assertEqual(vol.volume.dependent_names, [])
            self.assertEqual(op1.pvolume.dependent_names, [op1.name])
            self.assertEqual(op2.dependent_names, [op1.name])
        
        kfp.compiler.Compiler()._compile(my_pipeline) 

# 需要导入模块: from kfp import dsl [as 别名]
# 或者: from kfp.dsl import ContainerOp [as 别名]
def test_after_method(self):
        """Test the after method."""
        def my_pipeline():
            op1 = ContainerOp(name="op1", image="image")
            op2 = ContainerOp(name="op2", image="image").after(op1)
            op3 = ContainerOp(name="op3", image="image")
            vol1 = PipelineVolume(name="pipeline-volume")
            vol2 = vol1.after(op1)
            vol3 = vol2.after(op2)
            vol4 = vol3.after(op1, op2)
            vol5 = vol4.after(op3)

            self.assertEqual(vol1.dependent_names, [])
            self.assertEqual(vol2.dependent_names, [op1.name])
            self.assertEqual(vol3.dependent_names, [op2.name])
            self.assertEqual(sorted(vol4.dependent_names), [op1.name, op2.name])
            self.assertEqual(sorted(vol5.dependent_names),
                            [op1.name, op2.name, op3.name])

        kfp.compiler.Compiler()._compile(my_pipeline) 

# 需要导入模块: from kfp import dsl [as 别名]
# 或者: from kfp.dsl import ContainerOp [as 别名]
def volume_pipeline():
  op1 = dsl.ContainerOp(
      name='download',
      image='google/cloud-sdk',
      command=['sh', '-c'],
      arguments=['ls | tee /tmp/results.txt'],
      file_outputs={'downloaded': '/tmp/results.txt'}) \
    .add_volume(k8s_client.V1Volume(name='gcp-credentials',
                                   secret=k8s_client.V1SecretVolumeSource(
                                       secret_name='user-gcp-sa'))) \
    .add_volume_mount(k8s_client.V1VolumeMount(
      mount_path='/secret/gcp-credentials', name='gcp-credentials')) \
    .add_env_variable(k8s_client.V1EnvVar(
      name='GOOGLE_APPLICATION_CREDENTIALS',
      value='/secret/gcp-credentials/user-gcp-sa.json')) \
    .add_env_variable(k8s_client.V1EnvVar(name='Foo', value='bar'))
  op2 = dsl.ContainerOp(
      name='echo',
      image='library/bash',
      command=['sh', '-c'],
      arguments=['echo %s' % op1.output]) 

# 需要导入模块: from kfp import dsl [as 别名]
# 或者: from kfp.dsl import ContainerOp [as 别名]
def pipelineparams_pipeline(tag: str = 'latest', sleep_ms: int = 10):

    echo = dsl.Sidecar(
        name='echo',
        image='hashicorp/http-echo:%s' % tag,
        args=['-text="hello world"'],
    )

    op1 = dsl.ContainerOp(
        name='download',
        image='busybox:%s' % tag,
        command=['sh', '-c'],
        arguments=['sleep %s; wget localhost:5678 -O /tmp/results.txt' % sleep_ms],
        sidecars=[echo],
        file_outputs={'downloaded': '/tmp/results.txt'})

    op2 = dsl.ContainerOp(
        name='echo',
        image='library/bash',
        command=['sh', '-c'],
        arguments=['echo $MSG %s' % op1.output])
    
    op2.container.add_env_variable(V1EnvVar(name='MSG', value='pipelineParams: ')) 

# 需要导入模块: from kfp import dsl [as 别名]
# 或者: from kfp.dsl import ContainerOp [as 别名]
def pipeline(my_pipe_param: int = 10):
    loop_args = [{'a': 1, 'b': 2}, {'a': 10, 'b': 20}]
    with dsl.ParallelFor(loop_args) as item:
        op1 = dsl.ContainerOp(
            name="my-in-coop1",
            image="library/bash:4.4.23",
            command=["sh", "-c"],
            arguments=["echo op1 %s %s" % (item.a, my_pipe_param)],
        )

        op2 = dsl.ContainerOp(
            name="my-in-coop2",
            image="library/bash:4.4.23",
            command=["sh", "-c"],
            arguments=["echo op2 %s" % item.b],
        )

    op_out = dsl.ContainerOp(
        name="my-out-cop",
        image="library/bash:4.4.23",
        command=["sh", "-c"],
        arguments=["echo %s" % my_pipe_param],
    ) 

# 需要导入模块: from kfp import dsl [as 别名]
# 或者: from kfp.dsl import ContainerOp [as 别名]
def pipeline():
    op0 = dsl.ContainerOp(
        name="my-out-cop0",
        image='python:alpine3.6',
        command=["sh", "-c"],
        arguments=[
            'python -c "import json; import sys; json.dump([i for i in range(20, 31)], open(\'/tmp/out.json\', \'w\'))"'],
        file_outputs={'out': '/tmp/out.json'},
    )

    with dsl.ParallelFor(op0.output) as item:
        op1 = dsl.ContainerOp(
            name="my-in-cop1",
            image="library/bash:4.4.23",
            command=["sh", "-c"],
            arguments=["echo do output op1 item: %s" % item],
        )

    op_out = dsl.ContainerOp(
        name="my-out-cop2",
        image="library/bash:4.4.23",
        command=["sh", "-c"],
        arguments=["echo do output op2, outp: %s" % op0.output],
    ) 

# 需要导入模块: from kfp import dsl [as 别名]
# 或者: from kfp.dsl import ContainerOp [as 别名]
def pipeline():
    op0 = dsl.ContainerOp(
        name="my-out-cop0",
        image='python:alpine3.6',
        command=["sh", "-c"],
        arguments=['python -c "import json; import sys; json.dump([{\'a\': 1, \'b\': 2}, {\'a\': 10, \'b\': 20}], open(\'/tmp/out.json\', \'w\'))"'],
        file_outputs={'out': '/tmp/out.json'},
    )

    with dsl.ParallelFor(op0.output) as item:
        op1 = dsl.ContainerOp(
            name="my-in-cop1",
            image="library/bash:4.4.23",
            command=["sh", "-c"],
            arguments=["echo do output op1 item.a: %s" % item.a],
        )

    op_out = dsl.ContainerOp(
        name="my-out-cop2",
        image="library/bash:4.4.23",
        command=["sh", "-c"],
        arguments=["echo do output op2, outp: %s" % op0.output],
    ) 

# 需要导入模块: from kfp import dsl [as 别名]
# 或者: from kfp.dsl import ContainerOp [as 别名]
def pipeline(loopidy_doop: dict = [{'a': 1, 'b': 2}, {'a': 10, 'b': 20}]):
    op0 = dsl.ContainerOp(
        name="my-out-cop0",
        image='python:alpine3.6',
        command=["sh", "-c"],
        arguments=[
            'python -c "import json; import sys; json.dump([i for i in range(20, 31)], open(\'/tmp/out.json\', \'w\'))"'],
        file_outputs={'out': '/tmp/out.json'},
    )

    with dsl.ParallelFor(loopidy_doop) as item:
        op1 = dsl.ContainerOp(
            name="my-in-cop1",
            image="library/bash:4.4.23",
            command=["sh", "-c"],
            arguments=["echo no output global op1, item.a: %s" % item.a],
        ).after(op0)

    op_out = dsl.ContainerOp(
        name="my-out-cop2",
        image="library/bash:4.4.23",
        command=["sh", "-c"],
        arguments=["echo no output global op2, outp: %s" % op0.output],
    ) 

# 需要导入模块: from kfp import dsl [as 别名]
# 或者: from kfp.dsl import ContainerOp [as 别名]
def test_set_parallelism(self):
    """Test a pipeline with parallelism limits."""
    def some_op():
        return dsl.ContainerOp(
            name='sleep',
            image='busybox',
            command=['sleep 1'],
        )

    @dsl.pipeline()
    def some_pipeline():
      some_op()
      some_op()
      some_op()
      dsl.get_pipeline_conf().set_parallelism(1)

    workflow_dict = kfp.compiler.Compiler()._compile(some_pipeline)
    self.assertEqual(workflow_dict['spec']['parallelism'], 1) 

# 需要导入模块: from kfp import dsl [as 别名]
# 或者: from kfp.dsl import ContainerOp [as 别名]
def test_op_transformers(self):
    def some_op():
      return dsl.ContainerOp(
          name='sleep',
          image='busybox',
          command=['sleep 1'],
      )

    @dsl.pipeline(name='some_pipeline')
    def some_pipeline():
      task1 = some_op()
      task2 = some_op()
      task3 = some_op()

      dsl.get_pipeline_conf().op_transformers.append(lambda op: op.set_retry(5))

    workflow_dict = compiler.Compiler()._compile(some_pipeline)
    for template in workflow_dict['spec']['templates']:
      container = template.get('container', None)
      if container:
        self.assertEqual(template['retryStrategy']['limit'], 5) 

# 需要导入模块: from kfp import dsl [as 别名]
# 或者: from kfp.dsl import ContainerOp [as 别名]
def test_image_pull_policy(self):
    def some_op():
      return dsl.ContainerOp(
          name='sleep',
          image='busybox',
          command=['sleep 1'],
      )

    @dsl.pipeline(name='some_pipeline')
    def some_pipeline():
      task1 = some_op()
      task2 = some_op()
      task3 = some_op()

      dsl.get_pipeline_conf().set_image_pull_policy(policy="Always")
    workflow_dict = compiler.Compiler()._compile(some_pipeline)
    for template in workflow_dict['spec']['templates']:
      container = template.get('container', None)
      if container:
        self.assertEqual(template['container']['imagePullPolicy'], "Always")