Python Shell.scp方法代码示例

# 需要导入模块: from cloudmesh_client.common.Shell import Shell [as 别名]
# 或者: from cloudmesh_client.common.Shell.Shell import scp [as 别名]

#.........这里部分代码省略.........
            echo 'MPIRUN_NOALLOCATE:' $MPIRUN_NOALLOCATE
            echo 'MPIRUN_NOFREE:' $MPIRUN_NOFREE
            echo 'SLURM_NTASKS_PER_CORE:' $SLURM_NTASKS_PER_CORE
            echo 'SLURM_NTASKS_PER_NODE:' $SLURM_NTASKS_PER_NODE
            echo 'SLURM_NTASKS_PER_SOCKET:' $SLURM_NTASKS_PER_SOCKET
            echo 'SLURM_RESTART_COUNT:' $SLURM_RESTART_COUNT
            echo 'SLURM_SUBMIT_DIR:' $SLURM_SUBMIT_DIR
            echo 'MPIRUN_PARTITION:' $MPIRUN_PARTITION
            d=$(date)
            echo \"#CLOUDMESH: status, start, $d\"
            srun -l echo \"#CLOUDMESH: status, start, $d\"
            srun -l {command}
            d=$(date)
            srun -l echo \"#CLOUDMESH: status, finished, $d\"
            d=$(date)
            echo \"#CLOUDMESH: status, finished, $d\"
            """
        ).format(**data).replace("\r\n", "\n").strip()

        _from = Config.path_expand('~/.cloudmesh/{script_name}'.format(**data))
        _to = '{cluster}:{remote_experiment_dir}'.format(**data)
        data["from"] = _from
        data["to"] = _to
        data["script"] = script
        # write the script to local

        # print(_from)
        # print(_to)

        with open(_from, 'w') as local_file:
            local_file.write(script)

        # copy to remote host
        Shell.scp(_from, _to)

        # delete local file
        # Shell.execute('rm', _from)
        # import sys; sys.exit()

        # run the sbatch command

        cmd = 'sbatch {remote_experiment_dir}/{script_name}'.format(**data)
        data["cmd"] = cmd

        # print ("CMD>", cmd)
        result = Shell.ssh(cluster, cmd)

        data["output"] = result

        # find id
        for line in result.split("\n"):
            # print ("LLL>", line)
            if "Submitted batch job" in line:
                data["job_id"] = int(line.replace("Submitted batch job ", "").strip())
                break

        #
        # HACK, should not depend on Model.py
        #

        # from cloudmesh_client.db.model import BATCHJOB
        # name = ""
        # BATCHJOB(name,
        #         cluster=data["cluster"],
        #         id=data["id"],
        #         script=data["script"]) # has user and username which seems wrong

# 需要导入模块: from cloudmesh_client.common.Shell import Shell [as 别名]
# 或者: from cloudmesh_client.common.Shell.Shell import scp [as 别名]
    def remote(cls, host, force=False):
        """

        TODO: there is a bug in the instalation of kilo the openrc file on
        the remote machine is not called openrc.sh but contains username and
        project number.

        :param host: the remote host
        :param force:
        :return:
        """

        config = ConfigDict("cloudmesh.yaml")

        host_spec = config["cloudmesh.clouds." + host]
        host_credentials = host_spec["credentials"]

        if 'cm_openrc' in host_spec:
            Console.ok("looking for openrc")
        else:
            Console.error("no cm_openrc specified in the host")
            return

        hostname = config["cloudmesh.clouds." + host + ".cm_host"]
        Console.ok("fetching information from {:}  ...".format(host))

        openrc = host_spec["cm_openrc"]

        directory = os.path.dirname(openrc)
        base = os.path.basename(openrc)

        _from_dir = "{:}:{:}".format(hostname, directory + "/*").replace("~/", "")
        # _to_dir = os.path.dirname(Config.path_expand(directory))
        # FIX: Issues with path expanding on Windows
        _to_dir = os.path.realpath(
            os.path.expanduser(directory)
        )

        '''
        In Windows, SCP fails with path such as C:\\Users\\...,
            and passes with '~/.cloudmesh/...'
        But on Linux machines, it fails with ~/.cloudmesh/...
            and passes with /home/user/...
        Hence, adding OS check below for SCP copy directory
        '''
        os_type = platform.system().lower()
        if 'windows' not in os_type:
            directory = _to_dir

        # FIX: fix for scp not working on Windows, because scp does not
        # understand
        # paths in format: "C:/Users/<>", rather expects "~/.cloudmesh/<>"
        # openrc_file = Config.path_expand(openrc)
        openrc_file = os.path.realpath(
            os.path.expanduser(openrc)
        )
        print("From:  ", _from_dir)
        print("To:    ", _to_dir)
        print("Openrc:", openrc_file)

        cls.make_dir(_to_dir)
        r = ""
        Console.ok("Reading rc file from {}".format(host))
        try:
            r = Shell.scp('-r', _from_dir, directory)
        except Exception as e:
            print(e)
            return

        #
        # TODO: the permission are not yet right
        #
        os.chmod(_to_dir, 0o700)
        for root, dirs, _ in os.walk(_to_dir):
            for d in dirs:
                os.chmod(os.path.join(root, d), 0o700)
        #
        # END PERMISSION
        #

        with open(openrc_file, 'r') as f:
            lines = f.read().split("\n")

        config = ConfigDict("cloudmesh.yaml")
        for line in lines:
            if line.strip().startswith("export"):
                line = line.replace("export ", "")
                key, value = line.split("=", 1)
                config["cloudmesh"]["clouds"][host]["credentials"][key] = value
        host_spec = config["cloudmesh"]["clouds"][host]
        credentials = host_spec["credentials"]

        if "cm_openrc" in host_spec:
            openrc = host_spec["cm_openrc"]
            for attribute in credentials:
                if attribute in openrc:
                    openrc.replace(attribute, credentials[attribute])
        config.save()
        config = ConfigDict("cloudmesh.yaml")
        return config["cloudmesh"]["clouds"][host]["credentials"]