Python LOGGER.error方法代码示例

# 需要导入模块: from common import LOGGER [as 别名]
# 或者: from common.LOGGER import error [as 别名]
def main():
    parser = argparse.ArgumentParser("Start a number of CLEAN servers")
    parser.add_argument("-a", "--ami_id", help="the AMI id to use")
    parser.add_argument("-i", "--instance_type", required=True, help="the instance type to use")
    parser.add_argument("-c", "--created_by", help="the username to use")
    parser.add_argument("-n", "--name", required=True, help="the instance name to use")
    parser.add_argument("-s", "--spot_price", type=float, help="the spot price to use")
    parser.add_argument("-b", "--bash_script", help="the bash script to use")
    parser.add_argument("-p", "--processes", type=int, default=1, help="the number of processes to run")
    parser.add_argument("snapshots", nargs="+", help="the snapshots to use")

    args = vars(parser.parse_args())

    corrected_args = check_args(args)
    if corrected_args is None:
        LOGGER.error("The arguments are incorrect: {0}".format(args))
    else:
        start_server(
            args["processes"],
            corrected_args["ami_id"],
            corrected_args["user_data"],
            corrected_args["setup_disks"],
            args["instance_type"],
            args["snapshots"],
            corrected_args["created_by"],
            args["name"],
            corrected_args["instance_details"],
            corrected_args["spot_price"],
        )

# 需要导入模块: from common import LOGGER [as 别名]
# 或者: from common.LOGGER import error [as 别名]
def main():
    parser = argparse.ArgumentParser('Start a number of CLEAN servers')
    parser.add_argument('-a', '--ami_id', help='the AMI id to use')
    parser.add_argument('-i', '--instance_type', required=True, help='the instance type to use')
    parser.add_argument('-c', '--created_by', help='the username to use')
    parser.add_argument('-n', '--name', required=True, help='the instance name to use')
    parser.add_argument('-s', '--spot_price', type=float, help='the spot price to use')
    parser.add_argument('-b', '--bash_script', help='the bash script to use')
    parser.add_argument('-e', '--ebs', type=int, help='the size in GB of any EBS volume')
    parser.add_argument('bottom_frequency', help='The bottom frequency')
    parser.add_argument('frequency_range', help='the range of frequencies')
    parser.add_argument('obs_id', help='the observation id')

    args = vars(parser.parse_args())

    corrected_args = check_args(args)
    if corrected_args is None:
        LOGGER.error('The arguments are incorrect: {0}'.format(args))
    else:
        start_servers(
            corrected_args['ami_id'],
            corrected_args['user_data'],
            corrected_args['setup_disks'],
            args['instance_type'],
            make_safe_filename(args['obs_id']),
            corrected_args['created_by'],
            args['name'],
            corrected_args['instance_details'],
            corrected_args['spot_price'],
            args['ebs'],
            args['bottom_frequency'],
            args['frequency_range'])

# 需要导入模块: from common import LOGGER [as 别名]
# 或者: from common.LOGGER import error [as 别名]
def main():
    parser = argparse.ArgumentParser('Start a number of CLEAN servers')
    parser.add_argument('-a', '--ami_id', help='the AMI id to use')
    parser.add_argument('-i', '--instance_type', required=True, help='the instance type to use')
    parser.add_argument('-c', '--created_by', help='the username to use')
    parser.add_argument('-n', '--name', required=True, help='the instance name to use')
    parser.add_argument('-s', '--spot_price', type=float, help='the spot price to use')
    parser.add_argument('-b', '--bash_script', help='the bash script to use')
    parser.add_argument('-p', '--processes', type=int, default=1, help='the number of processes to run')
    parser.add_argument('frequencies', nargs='+', help='the frequencies to use (vis_14XX~14YY')

    args = vars(parser.parse_args())

    corrected_args = check_args(args)
    if corrected_args is None:
        LOGGER.error('The arguments are incorrect: {0}'.format(args))
    else:
        start_servers(
            args['processes'],
            corrected_args['ami_id'],
            corrected_args['user_data'],
            corrected_args['setup_disks'],
            args['instance_type'],
            args['frequencies'],
            corrected_args['created_by'],
            args['name'],
            corrected_args['instance_details'],
            corrected_args['spot_price'])

# 需要导入模块: from common import LOGGER [as 别名]
# 或者: from common.LOGGER import error [as 别名]
    def __call__(self):
        """
        Actually run the job
        """
        # Get the name of the volume
        ec2_helper = EC2Helper()
        iops = None
        if self._instance_details.iops_support:
            iops = 500

        zone = ec2_helper.get_cheapest_spot_price(self._instance_type, self._spot_price)
        if zone is not None:
            volume, snapshot_name = ec2_helper.create_volume(self._snapshot_id, zone, iops=iops)
            LOGGER.info('obs_id: {0}, volume_name: {1}'.format(self._obs_id, snapshot_name))
            user_data_mime = self.get_mime_encoded_user_data(volume.id)

            if self._spot_price is not None:
                ec2_helper.run_spot_instance(
                    self._ami_id,
                    self._spot_price,
                    user_data_mime,
                    self._instance_type,
                    volume.id,
                    self._created_by,
                    '{1}-{2}-{0}'.format(self._name, snapshot_name, self._counter),
                    self._instance_details,
                    zone,
                    ephemeral=True)
        else:
            LOGGER.error('Cannot get a spot instance of {0} for ${1}'.format(self._instance_type, self._spot_price))

# 需要导入模块: from common import LOGGER [as 别名]
# 或者: from common.LOGGER import error [as 别名]
def main():
    parser = argparse.ArgumentParser('Start a number of CVEL servers')
    parser.add_argument('-a', '--ami_id', help='the AMI id to use')
    parser.add_argument('-i', '--instance_type', required=True, help='the instance type to use')
    parser.add_argument('-c', '--created_by', help='the username to use')
    parser.add_argument('-n', '--name', required=True, help='the instance name to use')
    parser.add_argument('-s', '--spot_price', type=float, help='the spot price to use')
    parser.add_argument('-b', '--bash_script', help='the bash script to use')
    parser.add_argument('-p', '--processes', type=int, default=1, help='the number of processes to run')
    parser.add_argument('-fc', '--frequency_channels', type=int, default=28, help='how many frequency channels per AWS instance')
    parser.add_argument('--force', action='store_true', default=False, help='proceed with a frequency band even if we already have it')

    parser.add_argument('obs_ids', nargs='+', help='the ids of the observation')

    args = vars(parser.parse_args())

    corrected_args = check_args(args)
    if corrected_args is None:
        LOGGER.error('The arguments are incorrect: {0}'.format(args))
    else:
        start_servers(
            args['processes'],
            corrected_args['ami_id'],
            corrected_args['user_data'],
            corrected_args['setup_disks'],
            args['instance_type'],
            corrected_args['obs_ids'],
            corrected_args['created_by'],
            args['name'],
            corrected_args['instance_details'],
            corrected_args['spot_price'],
            args['frequency_channels'],
            args['force'])

# 需要导入模块: from common import LOGGER [as 别名]
# 或者: from common.LOGGER import error [as 别名]
def start_servers(
        ami_id,
        user_data,
        setup_disks,
        instance_type,
        obs_id,
        created_by,
        name,
        instance_details,
        spot_price,
        ebs,
        bottom_frequency,
        frequency_range):
    LOGGER.info('obs_id: {0}, bottom_frequency: {1}, frequency_range: {2}'.format(obs_id, bottom_frequency, frequency_range))
    ec2_helper = EC2Helper()
    zone = ec2_helper.get_cheapest_spot_price(instance_type, spot_price)

    if zone is not None:
        # Swap size
        if ebs is None:
            swap_size = 1
        else:
            ephemeral_size = instance_details.number_disks * instance_details.size
            swap_size = min(int(ephemeral_size * 0.75), 16)

        user_data_mime = get_mime_encoded_user_data(
            user_data,
            obs_id,
            setup_disks,
            bottom_frequency,
            frequency_range,
            swap_size
        )
        LOGGER.info('{0}'.format(user_data_mime))

        ec2_helper.run_spot_instance(
            ami_id,
            spot_price,
            user_data_mime,
            instance_type,
            None,
            created_by,
            name + '- {0}'.format(obs_id),
            instance_details=instance_details,
            zone=zone,
            ebs_size=ebs,
            number_ebs_volumes=4,
            ephemeral=True)
    else:
        LOGGER.error('Cannot get a spot instance of {0} for ${1}'.format(instance_type, spot_price))

# 需要导入模块: from common import LOGGER [as 别名]
# 或者: from common.LOGGER import error [as 别名]
def check_args(args):
    """
    Check the arguments and prompt for new ones
    """
    map_args = {}

    if args['obs_ids'] is None:
        return None
    elif len(args['obs_ids']) == 1 and args['obs_ids'][0] == '*':
        map_args['obs_ids'] = OBS_IDS.keys()
    else:
        map_args['obs_ids'] = args['obs_ids']

    if args['instance_type'] is None:
        return None

    if args['name'] is None:
        return None

    instance_details = AWS_INSTANCES.get(args['instance_type'])
    if instance_details is None:
        LOGGER.error('The instance type {0} is not supported.'.format(args['instance_type']))
        return None
    else:
        LOGGER.info(
            'instance: {0}, vCPU: {1}, RAM: {2}GB, Disks: {3}x{4}GB, IOPS: {5}'.format(
                args['instance_type'],
                instance_details.vCPU,
                instance_details.memory,
                instance_details.number_disks,
                instance_details.size,
                instance_details.iops_support))

    map_args.update({
        'ami_id': args['ami_id'] if args['ami_id'] is not None else AWS_AMI_ID,
        'created_by': args['created_by'] if args['created_by'] is not None else getpass.getuser(),
        'spot_price': args['spot_price'] if args['spot_price'] is not None else None,
        'user_data': get_script(args['bash_script'] if args['bash_script'] is not None else BASH_SCRIPT_CVEL),
        'setup_disks': get_script(BASH_SCRIPT_SETUP_DISKS),
        'instance_details': instance_details,
    })

    return map_args

# 需要导入模块: from common import LOGGER [as 别名]
# 或者: from common.LOGGER import error [as 别名]
def check_args(args):
    """
    Check the arguments and prompt for new ones
    """
    map_args = {}

    if args["snapshots"] is None:
        return None

    if args["instance_type"] is None:
        return None

    if args["name"] is None:
        return None

    instance_details = AWS_INSTANCES.get(args["instance_type"])
    if instance_details is None:
        LOGGER.error("The instance type {0} is not supported.".format(args["instance_type"]))
        return None
    else:
        LOGGER.info(
            "instance: {0}, vCPU: {1}, RAM: {2}GB, Disks: {3}x{4}GB, IOPS: {5}".format(
                args["instance_type"],
                instance_details.vCPU,
                instance_details.memory,
                instance_details.number_disks,
                instance_details.size,
                instance_details.iops_support,
            )
        )

    map_args.update(
        {
            "ami_id": args["ami_id"] if args["ami_id"] is not None else AWS_AMI_ID,
            "created_by": args["created_by"] if args["created_by"] is not None else getpass.getuser(),
            "spot_price": args["spot_price"] if args["spot_price"] is not None else None,
            "user_data": get_script(args["bash_script"] if args["bash_script"] is not None else BASH_SCRIPT_CLEAN_ALL),
            "setup_disks": get_script(BASH_SCRIPT_SETUP_DISKS),
            "instance_details": instance_details,
        }
    )
    return map_args

# 需要导入模块: from common import LOGGER [as 别名]
# 或者: from common.LOGGER import error [as 别名]
def start_servers(
        ami_id,
        user_data,
        setup_disks,
        instance_type,
        obs_id,
        created_by,
        name,
        instance_details,
        spot_price):

    snapshot_id = OBS_IDS.get(obs_id)
    if snapshot_id is None:
        LOGGER.warning('The obs-id: {0} does not exist in the settings file')
    else:
        ec2_helper = EC2Helper()
        iops = None
        if instance_details.iops_support:
            iops = 500

        zone = ec2_helper.get_cheapest_spot_price(instance_type, spot_price)
        if zone is not None:
            volume, snapshot_name = ec2_helper.create_volume(snapshot_id, zone, iops=iops)
            LOGGER.info('obs_id: {0}, volume_name: {1}'.format(obs_id, snapshot_name))
            now = datetime.datetime.now()
            user_data_mime = get_mime_encoded_user_data(volume.id, setup_disks, user_data, now.strftime('%Y-%m-%dT%H-%M-%S'))

            if spot_price is not None:
                ec2_helper.run_spot_instance(
                    ami_id,
                    spot_price,
                    user_data_mime,
                    instance_type,
                    volume.id,
                    created_by,
                    '{1}-{0}'.format(name, snapshot_name),
                    instance_details,
                    zone,
                    ephemeral=True)
        else:
            LOGGER.error('Cannot get a spot instance of {0} for ${1}'.format(instance_type, spot_price))

# 需要导入模块: from common import LOGGER [as 别名]
# 或者: from common.LOGGER import error [as 别名]
    def __call__(self):
        """
        Actually run the job
        """
        LOGGER.info('frequency_id: {0}'.format(self._frequency_id))
        ec2_helper = EC2Helper()
        zone = ec2_helper.get_cheapest_spot_price(self._instance_type, self._spot_price)

        if zone is not None:
            user_data_mime = self.get_mime_encoded_user_data()
            LOGGER.info('{0}'.format(user_data_mime))

            ec2_helper.run_spot_instance(
                self._ami_id,
                self._spot_price,
                user_data_mime,
                self._instance_type, None,
                self._created_by,
                '{0}-{1}'.format(self._frequency_id, self._name),
                instance_details=self._instance_details,
                zone=zone,
                ephemeral=True)
        else:
            LOGGER.error('Cannot get a spot instance of {0} for ${1}'.format(self._instance_type, self._spot_price))

# 需要导入模块: from common import LOGGER [as 别名]
# 或者: from common.LOGGER import error [as 别名]
def start_server(
    processes,
    ami_id,
    user_data,
    setup_disks,
    instance_type,
    snapshot_ids,
    created_by,
    name,
    instance_details,
    spot_price,
):

    LOGGER.info("snapshot_ids: {0}".format(snapshot_ids))
    ec2_helper = EC2Helper()
    zone = ec2_helper.get_cheapest_spot_price(instance_type, spot_price)

    if zone is not None:
        # Get the snapshot details
        connection = ec2_helper.ec2_connection
        snapshot_details = {}
        for snapshot in connection.get_all_snapshots(owner="self"):
            name = snapshot.tags.get("Name")
            if name is None:
                LOGGER.info("Looking at {0} - None".format(snapshot.id))
            elif snapshot.status == "completed":
                LOGGER.info("Looking at {0} - {1}".format(snapshot.id, snapshot.tags["Name"]))
                if snapshot.tags["Name"].endswith("_FINAL_PRODUCTS"):
                    snapshot_details[snapshot.id] = snapshot
            else:
                LOGGER.info(
                    "Looking at {0} - {1} which is {2}".format(snapshot.id, snapshot.tags["Name"], snapshot.status)
                )

        # Create the volumes
        bdm = blockdevicemapping.BlockDeviceMapping()

        # The ephemeral disk
        xvdb = BlockDeviceType()
        xvdb.ephemeral_name = "ephemeral0"
        bdm["/dev/xvdb"] = xvdb

        if instance_details.number_disks == 2:
            xvdc = BlockDeviceType()
            xvdc.ephemeral_name = "ephemeral1"
            bdm["/dev/xvdc"] = xvdc

        # The Scratch space
        for disks in range(0, SCRATCH_DISKS):
            xvd_n = blockdevicemapping.EBSBlockDeviceType(delete_on_termination=True)
            xvd_n.size = int(SCRATCH_DISK_SIZE)  # size in Gigabytes
            xvd_n.volume_type = "gp2"
            last_char = chr(ord("d") + disks)
            bdm["/dev/xvd" + last_char] = xvd_n

        disks = 0
        for snapshot_id in snapshot_ids:
            snapshot_detail = snapshot_details.get(snapshot_id)
            xvd_n = blockdevicemapping.EBSBlockDeviceType(delete_on_termination=True)
            xvd_n.size = int(snapshot_detail.volume_size)  # size in Gigabytes
            xvd_n.volume_type = "gp2"
            xvd_n.snapshot_id = snapshot_id
            last_char = chr(ord("d") + disks + SCRATCH_DISKS)
            bdm["/dev/xvd" + last_char] = xvd_n
            disks += 1

        user_data_mime = get_mime_encoded_user_data(instance_details, setup_disks, user_data)
        LOGGER.info("{0}".format(user_data_mime))

        ec2_helper.run_spot_instance(
            ami_id,
            spot_price,
            user_data_mime,
            instance_type,
            None,
            created_by,
            "BIG Clean",
            instance_details=instance_details,
            zone=zone,
            ephemeral=True,
            bdm=bdm,
        )
    else:
        LOGGER.error("Cannot get a spot instance of {0} for ${1}".format(instance_type, spot_price))