Python types.Mount方法代码示例

# 需要导入模块: from docker import types [as 别名]
# 或者: from docker.types import Mount [as 别名]
def mounts(tmpdir_factory, flowetl_mounts_dir):
    """
    Various mount objects needed by containers
    """
    logs = tmpdir_factory.mktemp("logs")
    pgdata = tmpdir_factory.mktemp("pgdata")

    dags_mount = Mount("/opt/airflow/dags", f"{flowetl_mounts_dir}/dags", type="bind")

    logs_mount = Mount("/mounts/logs", str(logs), type="bind")
    flowetl_mounts = [dags_mount, logs_mount]

    data_mount = Mount(
        "/var/lib/postgresql/data", str(pgdata), type="bind", consistency="delegated"
    )
    files_mount = Mount(
        "/files",
        str(Path(__file__).parent.parent.parent / "mounts" / "files"),
        type="bind",
    )
    flowdb_mounts = [data_mount, files_mount]

    return {"flowetl": flowetl_mounts, "flowdb": flowdb_mounts} 

# 需要导入模块: from docker import types [as 别名]
# 或者: from docker.types import Mount [as 别名]
def _get_mounts(user_working_dir: str):
    # mount target dir needs to be absolute
    target_dir = default_workdir

    user_dir = Mount(target=target_dir,
                     source=user_working_dir,
                     type='bind')

    compss_dir = os.environ['HOME'] + '/.COMPSs'
    os.makedirs(compss_dir, exist_ok=True)

    compss_log_dir = Mount(target='/root/.COMPSs',
                           source=compss_dir,
                           type='bind')

    mounts = [user_dir, compss_log_dir]

    return mounts 

# 需要导入模块: from docker import types [as 别名]
# 或者: from docker.types import Mount [as 别名]
def run_docker_container(image: str, timeout: int = 300, command: Optional[str] = None, reraise: bool = False,
                         mount: Optional[Tuple[str, str]] = None, label: str = 'Docker', include_stderr: bool = True) -> str:
    container = None
    try:
        kwargs = {'mounts': [Mount(*mount, read_only=False, type='bind')]} if mount else {}
        client = docker.from_env()
        container = client.containers.run(image, command=command, network_disabled=True, detach=True, **kwargs)
        container.wait(timeout=timeout)
        return container.logs(stderr=include_stderr).decode()
    except ReadTimeout:
        logging.warning('[{}]: timeout while processing'.format(label))
        if reraise:
            raise
    except (DockerException, IOError):
        logging.warning('[{}]: encountered process error while processing'.format(label))
        if reraise:
            raise
    finally:
        if container:
            with suppress(DockerException):
                container.stop()
            container.remove() 

# 需要导入模块: from docker import types [as 别名]
# 或者: from docker.types import Mount [as 别名]
def build_mount(mount_spec):
    kwargs = {}
    if mount_spec.options:
        for option, sdk_name in mount_spec.options_map[mount_spec.type].items():
            if option in mount_spec.options:
                kwargs[sdk_name] = mount_spec.options[option]

    return Mount(
        type=mount_spec.type, target=mount_spec.target, source=mount_spec.source,
        read_only=mount_spec.read_only, consistency=mount_spec.consistency, **kwargs
    )

# Labels 

# 需要导入模块: from docker import types [as 别名]
# 或者: from docker.types import Mount [as 别名]
def MakeDockerRunParams(
      self,
      model_path: Text,
      needs_mount: bool) -> Dict[Text, Any]:
    """Make parameters for docker `client.containers.run`.

    Args:
      model_path: A path to the model.
      needs_mount: If True, model_path will be mounted to the container.

    Returns:
      A dictionary of docker run parameters.
    """
    result = dict(
        self._BASE_DOCKER_RUN_PARAMS,
        image=self._image)

    if needs_mount:
      # model_path should be a local directory. In order to make TF Serving see
      # the host model path, we need to mount model path volume to the
      # container.
      assert os.path.isdir(model_path), '{} does not exist'.format(model_path)
      container_model_path = tf_serving_flavor.make_model_path(
          model_base_path=self._DEFAULT_MODEL_BASE_PATH,
          model_name=self._model_name,
          version=1)
      result.update(
          environment=self.MakeEnvVars(),
          mounts=[
              docker_types.Mount(
                  type='bind',
                  target=container_model_path,
                  source=model_path,
                  read_only=True)
          ])
    else:
      # model_path is presumably a remote URI. TF Serving is able to pickup
      # model in remote directly using gfile, so all we need to do is setting
      # environment variables correctly.
      result.update(
          environment=self.MakeEnvVars(model_path=model_path))

    return result 

# 需要导入模块: from docker import types [as 别名]
# 或者: from docker.types import Mount [as 别名]
def make_linux_binaries(repo_root: Path) -> Set[Path]:
    """
    Create binaries for Linux in a Docker container.

    Args:
        repo_root: The path to the root of the repository.

    Returns:
        A set of paths to the built binaries.
    """
    client = docker.from_env(version='auto')
    dist_dir = repo_root / 'dist'
    assert not dist_dir.exists() or not set(dist_dir.iterdir())

    target_dir = '/e2e'
    code_mount = Mount(
        source=str(repo_root.absolute()),
        target=target_dir,
        type='bind',
    )

    cmd_in_container = [
        'pip',
        'install',
        '.[packaging]',
        '&&',
        'python',
        'admin/create_pyinstaller_binaries.py',
    ]
    command = 'bash -c "{cmd}"'.format(cmd=' '.join(cmd_in_container))

    container = client.containers.run(
        image='python:3.7',
        mounts=[code_mount],
        command=command,
        working_dir=target_dir,
        remove=True,
        detach=True,
    )
    for line in container.logs(stream=True):
        line = line.strip()
        LOGGER.info(line)

    status_code = container.wait()['StatusCode']
    assert status_code == 0
    return set(dist_dir.iterdir()) 

# 需要导入模块: from docker import types [as 别名]
# 或者: from docker.types import Mount [as 别名]
def start_scraper(docker_client, task, dns, host_workdir):
    config = task["config"]
    offliner = config["task_name"]
    container_name = scraper_container_name(task["_id"], offliner)

    # remove container should it exists (should not)
    try:
        remove_container(docker_client, container_name)
    except docker.errors.NotFound:
        pass

    logger.debug(f'pulling image {config["image"]["name"]}:{config["image"]["tag"]}')
    docker_image = pull_image(
        docker_client, config["image"]["name"], tag=config["image"]["tag"]
    )

    # where to mount volume inside scraper
    mount_point = config["mount_point"]

    # mounts will be attached to host's fs, not this one
    mounts = [Mount(str(mount_point), str(host_workdir), type="bind")]

    command = config["str_command"]
    cpu_shares = config["resources"]["cpu"] * DEFAULT_CPU_SHARE
    mem_limit = config["resources"]["memory"]
    disk_limit = config["resources"]["disk"]

    return run_container(
        docker_client,
        image=docker_image,
        command=command,
        cpu_shares=cpu_shares,
        mem_limit=mem_limit,
        dns=dns,
        detach=True,
        labels={
            "zimfarm": "",
            "zimscraper": "yes",
            "task_id": task["_id"],
            "tid": short_id(task["_id"]),
            "schedule_name": task["schedule_name"],
            RESOURCES_DISK_LABEL: str(disk_limit),
            "human.cpu": str(config["resources"]["cpu"]),
            "human.memory": format_size(mem_limit),
            "human.disk": format_size(disk_limit),
        },
        mem_swappiness=0,
        mounts=mounts,
        name=container_name,
        remove=False,  # scaper container will be removed once log&zim handled
    ) 

# 需要导入模块: from docker import types [as 别名]
# 或者: from docker.types import Mount [as 别名]
def start_task_worker(docker_client, task, webapi_uri, username, workdir, worker_name):
    container_name = task_container_name(task["_id"])

    # remove container should it exists (should not)
    try:
        remove_container(docker_client, container_name)
    except docker.errors.NotFound:
        pass

    image, tag = TASK_WORKER_IMAGE.rsplit(":", 1)
    if tag == "local":
        docker_image = get_image(docker_client, TASK_WORKER_IMAGE)
    else:
        logger.debug(f"pulling image {image}:{tag}")
        docker_image = pull_image(docker_client, image, tag=tag)

    # mounts will be attached to host's fs, not this one
    host_mounts = query_host_mounts(docker_client, workdir)
    host_task_workdir = str(host_mounts.get(workdir))
    host_docker_socket = str(host_mounts.get(DOCKER_SOCKET))
    host_private_key = str(host_mounts.get(PRIVATE_KEY))
    mounts = [
        Mount(str(workdir), host_task_workdir, type="bind"),
        Mount(str(DOCKER_SOCKET), host_docker_socket, type="bind", read_only=True),
        Mount(str(PRIVATE_KEY), host_private_key, type="bind", read_only=True),
    ]
    command = ["task-worker", "--task-id", task["_id"]]

    logger.debug(f"running {command}")
    return run_container(
        docker_client,
        image=docker_image,
        command=command,
        detach=True,
        environment={
            "USERNAME": username,
            "WORKDIR": str(workdir),
            "WEB_API_URI": webapi_uri,
            "UPLOAD_URI": UPLOAD_URI,
            "WORKER_NAME": worker_name,
            "ZIMFARM_DISK": os.getenv("ZIMFARM_DISK"),
            "ZIMFARM_CPUS": os.getenv("ZIMFARM_CPUS"),
            "ZIMFARM_MEMORY": os.getenv("ZIMFARM_MEMORY"),
            "DEBUG": os.getenv("DEBUG"),
            "USE_PUBLIC_DNS": "1" if USE_PUBLIC_DNS else "",
        },
        labels={
            "zimfarm": "",
            "zimtask": "yes",
            "task_id": task["_id"],
            "tid": short_id(task["_id"]),
            "schedule_name": task["schedule_name"],
        },
        mem_swappiness=0,
        mounts=mounts,
        name=container_name,
        remove=False,  # zimtask containers are pruned periodically
    ) 

# 需要导入模块: from docker import types [as 别名]
# 或者: from docker.types import Mount [as 别名]
def get_container_options(self, defaults):
        # Default environmental vars
        options = self.default_container_options()

        if not defaults:
            environment = dict()

            write_pretty_output("Provide configuration options for the THREDDS container or or press enter to "
                                "accept the defaults shown in square brackets: ")

            environment['TDM_PW'] = UserInputHelper.get_verified_password(
                prompt='TDM Password',
                default=options['environment']['TDM_PW'],
            )

            environment['TDS_HOST'] = UserInputHelper.get_input_with_default(
                prompt='TDS Host',
                default=options['environment']['TDS_HOST'],
            )

            environment['THREDDS_XMX_SIZE'] = UserInputHelper.get_input_with_default(
                prompt='TDS JVM Max Heap Size',
                default=options['environment']['THREDDS_XMX_SIZE'],
            )

            environment['THREDDS_XMS_SIZE'] = UserInputHelper.get_input_with_default(
                prompt='TDS JVM Min Heap Size',
                default=options['environment']['THREDDS_XMS_SIZE'],
            )

            environment['TDM_XMX_SIZE'] = UserInputHelper.get_input_with_default(
                prompt='TDM JVM Max Heap Size',
                default=options['environment']['TDM_XMX_SIZE'],
            )

            environment['TDM_XMS_SIZE'] = UserInputHelper.get_input_with_default(
                prompt='TDM JVM Min Heap Size',
                default=options['environment']['TDM_XMS_SIZE'],
            )

            options.update(environment=environment)

            mount_data_dir = UserInputHelper.get_valid_choice_input(
                prompt='Bind the THREDDS data directory to the host?',
                choices=['y', 'n'],
                default='y',
            )

            if mount_data_dir.lower() == 'y':
                tethys_home = get_tethys_home_dir()
                default_mount_location = os.path.join(tethys_home, 'thredds')
                thredds_data_volume = '/usr/local/tomcat/content/thredds'
                mount_location = UserInputHelper.get_valid_directory_input(
                    prompt='Specify location to bind the THREDDS data directory',
                    default=default_mount_location
                )
                mounts = [Mount(thredds_data_volume, mount_location, type='bind')]
                options['host_config'].update(mounts=mounts)

        return options