Python APIClient.build方法代码示例

# 需要导入模块: from docker import APIClient [as 别名]
# 或者: from docker.APIClient import build [as 别名]
class DockerBuilder(object):
    LATEST_IMAGE_TAG = 'latest'
    WORKDIR = '/code'

    def __init__(self,
                 build_job,
                 repo_path,
                 from_image,
                 copy_code=True,
                 build_steps=None,
                 env_vars=None,
                 dockerfile_name='Dockerfile'):
        self.build_job = build_job
        self.job_uuid = build_job.uuid.hex
        self.job_name = build_job.unique_name
        self.from_image = from_image
        self.image_name = get_image_name(self.build_job)
        self.image_tag = self.job_uuid
        self.folder_name = repo_path.split('/')[-1]
        self.repo_path = repo_path
        self.copy_code = copy_code

        self.build_path = '/'.join(self.repo_path.split('/')[:-1])
        self.build_steps = get_list(build_steps)
        self.env_vars = get_list(env_vars)
        self.dockerfile_path = os.path.join(self.build_path, dockerfile_name)
        self.polyaxon_requirements_path = self._get_requirements_path()
        self.polyaxon_setup_path = self._get_setup_path()
        self.docker = APIClient(version='auto')
        self.registry_host = None
        self.docker_url = None

    def get_tagged_image(self):
        return get_tagged_image(self.build_job)

    def check_image(self):
        return self.docker.images(self.get_tagged_image())

    def clean(self):
        # Clean dockerfile
        delete_path(self.dockerfile_path)

    def login(self, registry_user, registry_password, registry_host):
        try:
            self.docker.login(username=registry_user,
                              password=registry_password,
                              registry=registry_host,
                              reauth=True)
        except DockerException as e:
            _logger.exception('Failed to connect to registry %s\n', e)

    @staticmethod
    def _prepare_log_lines(log_line):
        raw = log_line.decode('utf-8').strip()
        raw_lines = raw.split('\n')
        log_lines = []
        for raw_line in raw_lines:
            try:
                json_line = json.loads(raw_line)

                if json_line.get('error'):
                    raise DockerBuilderError(str(json_line.get('error', json_line)))
                else:
                    if json_line.get('stream'):
                        log_lines.append('Build: {}'.format(json_line['stream'].strip()))
                    elif json_line.get('status'):
                        log_lines.append('Push: {} {}'.format(
                            json_line['status'],
                            json_line.get('progress')
                        ))
                    elif json_line.get('aux'):
                        log_lines.append('Push finished: {}'.format(json_line.get('aux')))
                    else:
                        log_lines.append(str(json_line))
            except json.JSONDecodeError:
                log_lines.append('JSON decode error: {}'.format(raw_line))
        return log_lines

    def _handle_logs(self, log_lines):
        publisher.publish_build_job_log(
            log_lines=log_lines,
            job_uuid=self.job_uuid,
            job_name=self.job_name
        )

    def _handle_log_stream(self, stream):
        log_lines = []
        last_emit_time = time.time()
        try:
            for log_line in stream:
                log_lines += self._prepare_log_lines(log_line)
                publish_cond = (
                    len(log_lines) == publisher.MESSAGES_COUNT or
                    (log_lines and time.time() - last_emit_time > publisher.MESSAGES_TIMEOUT)
                )
                if publish_cond:
                    self._handle_logs(log_lines)
                    log_lines = []
                    last_emit_time = time.time()
            if log_lines:
#.........这里部分代码省略.........

# 需要导入模块: from docker import APIClient [as 别名]
# 或者: from docker.APIClient import build [as 别名]
class BaseDockerBuilder(object):
    CHECK_INTERVAL = 10
    LATEST_IMAGE_TAG = 'latest'
    WORKDIR = '/code'

    def __init__(self,
                 repo_path,
                 from_image,
                 image_name,
                 image_tag,
                 copy_code=True,
                 in_tmp_repo=True,
                 build_steps=None,
                 env_vars=None,
                 dockerfile_name='Dockerfile'):
        # This will help create a unique tmp folder for dockerizer in case of concurrent jobs
        self.uuid = uuid.uuid4().hex
        self.from_image = from_image
        self.image_name = image_name
        self.image_tag = image_tag
        self.repo_path = repo_path
        self.folder_name = repo_path.split('/')[-1]
        self.copy_code = copy_code
        self.in_tmp_repo = in_tmp_repo
        if in_tmp_repo and copy_code:
            self.build_repo_path = self.create_tmp_repo()
        else:
            self.build_repo_path = self.repo_path

        self.build_path = '/'.join(self.build_repo_path.split('/')[:-1])
        self.build_steps = get_list(build_steps)
        self.env_vars = get_list(env_vars)
        self.dockerfile_path = os.path.join(self.build_path, dockerfile_name)
        self.polyaxon_requirements_path = self._get_requirements_path()
        self.polyaxon_setup_path = self._get_setup_path()
        self.docker = APIClient(version='auto')
        self.registry_host = None
        self.docker_url = None

    def get_tagged_image(self):
        return '{}:{}'.format(self.image_name, self.image_tag)

    def create_tmp_repo(self):
        # Create a tmp copy of the repo before starting the build
        return copy_to_tmp_dir(path=self.repo_path,
                               dir_name=os.path.join(self.uuid, self.image_tag, self.folder_name))

    def check_image(self):
        return self.docker.images(self.get_tagged_image())

    def clean(self):
        # Clean dockerfile
        delete_path(self.dockerfile_path)

        # Clean tmp dir if created
        if self.in_tmp_repo and self.copy_code:
            delete_tmp_dir(self.image_tag)

    def login(self, registry_user, registry_password, registry_host):
        try:
            self.docker.login(username=registry_user,
                              password=registry_password,
                              registry=registry_host,
                              reauth=True)
        except DockerException as e:
            _logger.exception('Failed to connect to registry %s\n', e)

    def _handle_logs(self, log_line):
        raise NotImplementedError

    def _check_pulse(self, check_pulse):
        """Checks if the job/experiment is still running.

        returns:
          * int: the updated check_pulse (+1) value
          * boolean: if the docker process should stop
        """
        raise NotImplementedError

    def _get_requirements_path(self):
        requirements_path = os.path.join(self.build_repo_path, 'polyaxon_requirements.txt')
        if os.path.isfile(requirements_path):
            return os.path.join(self.folder_name, 'polyaxon_requirements.txt')
        return None

    def _get_setup_path(self):
        setup_file_path = os.path.join(self.build_repo_path, 'polyaxon_setup.sh')
        has_setup = os.path.isfile(setup_file_path)
        if has_setup:
            st = os.stat(setup_file_path)
            os.chmod(setup_file_path, st.st_mode | stat.S_IEXEC)
            return os.path.join(self.folder_name, 'polyaxon_setup.sh')
        return None

    def render(self):
        docker_template = jinja2.Template(POLYAXON_DOCKER_TEMPLATE)
        return docker_template.render(
            from_image=self.from_image,
            polyaxon_requirements_path=self.polyaxon_requirements_path,
            polyaxon_setup_path=self.polyaxon_setup_path,
#.........这里部分代码省略.........

# 需要导入模块: from docker import APIClient [as 别名]
# 或者: from docker.APIClient import build [as 别名]
class DockerProxy:

    """ A wrapper over docker-py and some utility methods and classes. """

    LOG_TAG = "Docker "

    shell_commands = ["source"]

    class ImageBuildException(Exception):
        def __init__(self, message=None):
            super("Something went wrong while building docker container image.\n{0}".format(message))

    def __init__(self):
        self.client = Client(base_url=Constants.DOCKER_BASE_URL)
        self.build_count = 0
        logging.basicConfig(level=logging.DEBUG)

    @staticmethod
    def get_container_volume_from_working_dir(working_directory):
        import os
        return os.path.join("/home/ubuntu/", os.path.basename(working_directory))

    def create_container(self, image_str, working_directory=None, name=None,
                         port_bindings={Constants.DEFAULT_PUBLIC_WEBSERVER_PORT: ('127.0.0.1', 8080),
                                        Constants.DEFAULT_PRIVATE_NOTEBOOK_PORT: ('127.0.0.1', 8081)}):
        """Creates a new container with elevated privileges. Returns the container ID. Maps port 80 of container
        to 8080 of locahost by default"""

        docker_image = DockerImage.from_string(image_str)
        volume_dir = DockerProxy.get_container_volume_from_working_dir(working_directory)

        if name is None:
            import uuid
            random_str = str(uuid.uuid4())
            name = constants.Constants.MolnsDockerContainerNamePrefix + random_str[:8]
        image = docker_image.image_id if docker_image.image_id is not Constants.DockerNonExistentTag \
            else docker_image.image_tag

        logging.info("Using image {0}".format(image))
        import os
        if DockerProxy._verify_directory(working_directory) is False:
            if working_directory is not None:
                raise InvalidVolumeName("\n\nMOLNs uses certain reserved names for its configuration files in the "
                                        "controller environment, and unfortunately the provided name for working "
                                        "directory of the controller cannot be one of these. Please configure this "
                                        "controller again with a different volume name and retry. "
                                        "Here is the list of forbidden names: \n{0}"
                                        .format(Constants.ForbiddenVolumeNames))

            logging.warning(DockerProxy.LOG_TAG + "Unable to verify provided directory to use to as volume. Volume will NOT "
                                             "be created.")
            hc = self.client.create_host_config(privileged=True, port_bindings=port_bindings)
            container = self.client.create_container(image=image, name=name, command="/bin/bash", tty=True, detach=True,
                                                     ports=[Constants.DEFAULT_PUBLIC_WEBSERVER_PORT,
                                                            Constants.DEFAULT_PRIVATE_NOTEBOOK_PORT],
                                                     host_config=hc,
                                                     environment={"PYTHONPATH": "/usr/local/"})

        else:
            container_mount_point = '/home/ubuntu/{0}'.format(os.path.basename(working_directory))
            hc = self.client.create_host_config(privileged=True, port_bindings=port_bindings,
                                                binds={working_directory: {'bind': container_mount_point,
                                                                           'mode': 'rw'}})

            container = self.client.create_container(image=image, name=name, command="/bin/bash", tty=True, detach=True,
                                                     ports=[Constants.DEFAULT_PUBLIC_WEBSERVER_PORT,
                                                            Constants.DEFAULT_PRIVATE_NOTEBOOK_PORT],
                                                     volumes=container_mount_point, host_config=hc,
                                                     working_dir=volume_dir,
                                                     environment={"PYTHONPATH": "/usr/local/"})

        container_id = container.get("Id")

        return container_id

    # noinspection PyBroadException
    @staticmethod
    def _verify_directory(working_directory):
        import os
        if working_directory is None or os.path.basename(working_directory) in Constants.ForbiddenVolumeNames:
            return False
        try:
            if not os.path.exists(working_directory):
                os.makedirs(working_directory)
            return True
        except:
            return False

    def stop_containers(self, container_ids):
        """Stops given containers."""
        for container_id in container_ids:
            self.stop_container(container_id)

    def stop_container(self, container_id):
        """Stops the container with given ID."""
        self.client.stop(container_id)

    def container_status(self, container_id):
        """Checks if container with given ID running."""
        status = ProviderBase.STATUS_TERMINATED
#.........这里部分代码省略.........