Python logger_factory.EspaLogging类代码示例

def archive_log_files(order_id, product_id):
    """Archive the log files for the current job
    """

    logger = EspaLogging.get_logger(settings.PROCESSING_LOGGER)

    try:
        # Determine the destination path for the logs
        output_dir = Environment().get_distribution_directory()
        destination_path = os.path.join(output_dir, 'logs', order_id)
        # Create the path
        utilities.create_directory(destination_path)

        # Job log file
        logfile_path = EspaLogging.get_filename(settings.PROCESSING_LOGGER)
        full_logfile_path = os.path.abspath(logfile_path)
        log_name = os.path.basename(full_logfile_path)
        # Determine full destination
        destination_file = os.path.join(destination_path, log_name)
        # Copy it
        shutil.copyfile(full_logfile_path, destination_file)

        # Mapper log file
        full_logfile_path = os.path.abspath(MAPPER_LOG_FILENAME)
        final_log_name = '-'.join([MAPPER_LOG_PREFIX, order_id, product_id])
        final_log_name = '.'.join([final_log_name, 'log'])
        # Determine full destination
        destination_file = os.path.join(destination_path, final_log_name)
        # Copy it
        shutil.copyfile(full_logfile_path, destination_file)

    except Exception:
        # We don't care because we are at the end of processing
        # And if we are on the successful path, we don't care either
        logger.exception("Exception encountered and follows")

def set_product_error(server, order_id, product_id, processing_location):
    '''
    Description:
        Call the xmlrpc server routine to set a product request to error.
        Provides a sleep retry implementation to hopefully by-pass any errors
        encountered, so that we do not get requests that have failed, but
        show a status of processing.
    '''

    if server is not None:
        logger = EspaLogging.get_logger(settings.PROCESSING_LOGGER)

        attempt = 0
        sleep_seconds = settings.DEFAULT_SLEEP_SECONDS
        while True:
            try:
                # START - DEBUG
                if product_id is None:
                    logger.info("DEBUG: Product ID is [None]")
                else:
                    logger.info("DEBUG: Product ID is [%s]" % product_id)
                if order_id is None:
                    logger.info("DEBUG: Order ID is [None]")
                else:
                    logger.info("DEBUG: Order ID is [%s]" % order_id)
                if processing_location is None:
                    logger.info("DEBUG: Processing Location is [None]")
                else:
                    logger.info("DEBUG: Processing Location is [%s]"
                                % processing_location)
                # END - DEBUG

                logged_contents = \
                    EspaLogging.read_logger_file(settings.PROCESSING_LOGGER)

                status = server.set_scene_error(product_id, order_id,
                                                processing_location,
                                                logged_contents)

                if not status:
                    logger.critical("Failed processing xmlrpc call to"
                                    " set_scene_error")
                    return False

                break

            except Exception, e:
                logger.critical("Failed processing xmlrpc call to"
                                " set_scene_error")
                logger.exception("Exception encountered and follows")

                if attempt < settings.MAX_SET_SCENE_ERROR_ATTEMPTS:
                    sleep(sleep_seconds)  # sleep before trying again
                    attempt += 1
                    sleep_seconds = int(sleep_seconds * 1.5)
                    continue
                else:
                    return False

def untar_data(source_file, destination_directory):
    '''
    Description:
        Using tar extract the file contents into a destination directory.

    Notes:
        Works with '*.tar.gz' and '*.tar' files.
    '''

    logger = EspaLogging.get_logger(settings.PROCESSING_LOGGER)

    # If both source and destination are localhost we can just copy the data
    cmd = ' '.join(['tar', '--directory', destination_directory,
                    '-xvf', source_file])

    logger.info("Unpacking [%s] to [%s]"
                % (source_file, destination_directory))

    # Unpack the data and raise any errors
    output = ''
    try:
        output = utilities.execute_cmd(cmd)
    except Exception as e:
        logger.error("Failed to unpack data")
        raise e
    finally:
        if len(output) > 0:
            logger.info(output)

def reformat(metadata_filename, work_directory, input_format, output_format):
    '''
    Description:
      Re-format the bands to the specified format using our raw binary tools
      or gdal, whichever is appropriate for the task.

      Input espa:
          Output Formats: envi(espa), gtiff, and hdf
    '''

    logger = EspaLogging.get_logger(settings.PROCESSING_LOGGER)

    # Don't do anything if they match
    if input_format == output_format:
        return

    # Change to the working directory
    current_directory = os.getcwd()
    os.chdir(work_directory)

    try:
        # Convert from our internal ESPA/ENVI format to GeoTIFF
        if input_format == 'envi' and output_format == 'gtiff':
            gtiff_name = metadata_filename.rstrip('.xml')
            # Call with deletion of source files
            cmd = ' '.join(['convert_espa_to_gtif', '--del_src_files',
                            '--xml', metadata_filename,
                            '--gtif', gtiff_name])

            output = ''
            try:
                output = utilities.execute_cmd(cmd)

                # Rename the XML file back to *.xml from *_gtif.xml
                meta_gtiff_name = metadata_filename.split('.xml')[0]
                meta_gtiff_name = ''.join([meta_gtiff_name, '_gtif.xml'])

                os.rename(meta_gtiff_name, metadata_filename)
            except Exception, e:
                raise ee.ESPAException(ee.ErrorCodes.reformat,
                                       str(e)), None, sys.exc_info()[2]
            finally:
                if len(output) > 0:
                    logger.info(output)

            # Remove all the *.tfw files since gtiff was chosen a bunch may
            # be present
            files_to_remove = glob.glob('*.tfw')
            if len(files_to_remove) > 0:
                cmd = ' '.join(['rm', '-rf'] + files_to_remove)
                logger.info(' '.join(['REMOVING TFW DATA COMMAND:', cmd]))

                output = ''
                try:
                    output = utilities.execute_cmd(cmd)
                except Exception, e:
                    raise ee.ESPAException(ee.ErrorCodes.reformat,
                                           str(e)), None, sys.exc_info()[2]
                finally:

def get_landsat_metadata(work_dir, product_id):
    '''
    Description:
        Fixes potentially bad MTL file from Landsat and returns the Landsat
        metadata filename to use with the rest of the processing.
    '''
    logger = EspaLogging.get_logger(settings.PROCESSING_LOGGER)

    # Find the metadata file
    metadata_filename = ''

    # Save the current directory and change to the work directory
    current_directory = os.getcwd()
    os.chdir(work_dir)

    try:
        for meta_file in glob.glob('%s_MTL.*' % product_id):
            if ('old' not in meta_file and
                    not meta_file.startswith('lnd')):

                # Save the filename and break out of the directory loop
                metadata_filename = meta_file
                break

        if metadata_filename == '':
            msg = "Could not locate the Landsat MTL file in %s" % work_dir
            raise RuntimeError(msg)

        logger.info("Located MTL file:%s" % metadata_filename)

        # Backup the original file
        shutil.copy(metadata_filename, ''.join([metadata_filename, '.old']))

        file_data = list()
        # Read in the file and write it back out to get rid of binary
        # characters at the end of some of the GLS metadata files
        with open(metadata_filename, 'r') as metadata_fd:
            file_data = metadata_fd.readlines()

        data_buffer = StringIO()
        for line in file_data:
            data_buffer.write(line)
        fixed_data = data_buffer.getvalue()

        # Fix the stupid error where the filename has a bad extention
        if metadata_filename.endswith('.TIF'):
            metadata_filename = metadata_filename.replace('.TIF', '.txt')

        # Write the newly formatted file out
        with open(metadata_filename, 'w+') as metadata_fd:
            metadata_fd.write(fixed_data)

    finally:
        # Change back to the original directory
        os.chdir(current_directory)

    return metadata_filename

def distribute_product_local(product_name, source_path, packaging_path):

    logger = EspaLogging.get_logger(settings.PROCESSING_LOGGER)

    # Deliver the product files
    # Attempt X times sleeping between each attempt
    sleep_seconds = settings.DEFAULT_SLEEP_SECONDS
    max_number_of_attempts = settings.MAX_DISTRIBUTION_ATTEMPTS
    max_package_attempts = settings.MAX_PACKAGING_ATTEMPTS
    max_delivery_attempts = settings.MAX_DELIVERY_ATTEMPTS

    attempt = 0
    product_file = 'ERROR'
    cksum_file = 'ERROR'

    while True:
        try:
            # Package the product files to the online cache location
            # Attempt X times sleeping between each sub_attempt
            sub_attempt = 0
            while True:
                try:
                    (product_file, cksum_file,
                     local_cksum_value) = package_product(source_path,
                                                          packaging_path,
                                                          product_name)
                except Exception as e:
                    logger.exception("An exception occurred processing %s"
                                     % product_name)
                    if sub_attempt < max_package_attempts:
                        sleep(sleep_seconds)  # sleep before trying again
                        sub_attempt += 1
                        continue
                    else:
                        raise ee.ESPAException(ee.ErrorCodes.packaging_product,
                                               str(e)), None, sys.exc_info()[2]
                break

            # Always log where we placed the files
            logger.info("Delivered product to location %s"
                        " and checksum location %s" % (product_file,
                                                       cksum_file))
        except Exception as e:
            if attempt < max_number_of_attempts:
                sleep(sleep_seconds)  # sleep before trying again
                attempt += 1
                # adjust for next set
                sleep_seconds = int(sleep_seconds * 1.5)
                continue
            else:
                # May already be an ESPAException so don't override that
                raise e
        break

    return (product_file, cksum_file)

def transfer_file(source_host, source_file,
                  destination_host, destination_file,
                  source_username=None, source_pw=None,
                  destination_username=None, destination_pw=None):
    '''
    Description:
      Using cp/FTP/SCP transfer a file from a source location to a destination
      location.

    Notes:
      We are not doing anything significant here other then some logic and
      fallback to SCP if FTP fails.
    '''

    logger = EspaLogging.get_logger(settings.PROCESSING_LOGGER)

    logger.info("Transfering [%s:%s] to [%s:%s]"
                % (source_host, source_file,
                   destination_host, destination_file))

    # If both source and destination are localhost we can just copy the data
    if source_host == 'localhost' and destination_host == 'localhost':
        shutil.copyfile(source_file, destination_file)
        return

    # If both source and destination hosts are the same, we can use ssh to copy
    # the files locally on the remote host
    if source_host == destination_host:
        remote_copy_file_to_file(source_host, source_file, destination_file)
        return

    # Try FTP first before SCP if usernames and passwords are provided
    if source_username is not None and source_pw is not None:
        try:
            ftp_from_remote_location(source_username, source_pw, source_host,
                                     source_file, destination_file)
            return
        except Exception as e:
            logger.warning("FTP failures will attempt transfer using SCP")
            logger.warning("FTP Errors: %s" % str(e))

    elif destination_username is not None and destination_pw is not None:
        try:
            ftp_to_remote_location(destination_username, destination_pw,
                                   source_file, destination_host,
                                   destination_file)
            return
        except Exception as e:
            logger.warning("FTP failures will attempt transfer using SCP")
            logger.warning("FTP Errors: %s" % str(e))

    # As a last resort try SCP
    scp_transfer_file(source_host, source_file,
                      destination_host, destination_file)

def scp_transfer_file(source_host, source_file,
                      destination_host, destination_file):
    '''
    Description:
      Using SCP transfer a file from a source location to a destination
      location.

    Note:
      - It is assumed ssh has been setup for access between the localhost
        and destination system
      - If wild cards are to be used with the source, then the destination
        file must be a directory.  ***No checking is performed in this code***
    '''

    logger = EspaLogging.get_logger(settings.PROCESSING_LOGGER)

    if source_host == destination_host:
        msg = "source and destination host match unable to scp"
        logger.error(msg)
        raise Exception(msg)

    cmd = ['scp', '-q', '-o', 'StrictHostKeyChecking=no', '-c', 'arcfour',
           '-C']

    # Build the source portion of the command
    # Single quote the source to allow for wild cards
    if source_host == 'localhost':
        cmd.append(source_file)
    else:
        cmd.append("'%s:%s'" % (source_host, source_file))

    # Build the destination portion of the command
    if destination_host == 'localhost':
        cmd.append(destination_file)
    else:
        cmd.append('%s:%s' % (destination_host, destination_file))

    cmd = ' '.join(cmd)

    # Transfer the data and raise any errors
    output = ''
    try:
        output = utilities.execute_cmd(cmd)
    except Exception as e:
        if len(output) > 0:
            logger.info(output)
        logger.error("Failed to transfer data")
        raise e

    logger.info("Transfer complete - SCP")

def ftp_from_remote_location(username, pw, host, remotefile, localfile):
    '''
    Author: David Hill

    Date: 12/5/13

    Description:
      Transfers files from a remote location to the local machine using ftplib.

    Parameters:
      username = Username for ftp account
      pw = Password for ftp account
      host = The ftp server host
      remotefile = The file to transfer
      localfile = Full path to where the local file should be created.
                  (Parent directories must exist)

    Returns: None

    Errors: Raises Exception() in the event of error
    '''

    logger = EspaLogging.get_logger(settings.PROCESSING_LOGGER)

    # Make sure the src_file is absolute, otherwise ftp will choke
    if not remotefile.startswith('/'):
        remotefile = ''.join(['/', remotefile])

    pw = urllib2.unquote(pw)

    url = 'ftp://%s/%s' % (host, remotefile)

    logger.info("Transferring file from %s to %s" % (url, localfile))
    ftp = None
    try:
        with open(localfile, 'wb') as loc_file:
            def callback(data):
                loc_file.write(data)

            ftp = ftplib.FTP(host, timeout=60)
            ftp.login(user=username, passwd=pw)
            ftp.set_debuglevel(0)
            ftp.retrbinary(' '.join(['RETR', remotefile]), callback)

    finally:
        if ftp:
            ftp.quit()

    logger.info("Transfer complete - FTP")

def warp_image(source_file, output_file,
               base_warp_command=None,
               resample_method='near',
               pixel_size=None,
               no_data_value=None):
    '''
    Description:
      Executes the warping command on the specified source file
    '''

    logger = EspaLogging.get_logger(settings.PROCESSING_LOGGER)

    try:
        # Turn GDAL PAM off to prevent *.aux.xml files
        os.environ['GDAL_PAM_ENABLED'] = 'NO'

        cmd = copy.deepcopy(base_warp_command)

        # Resample method to use
        cmd.extend(['-r', resample_method])

        # Resize the pixels
        if pixel_size is not None:
            cmd.extend(['-tr', str(pixel_size), str(pixel_size)])

        # Specify the fill/nodata value
        if no_data_value is not None:
            cmd.extend(['-srcnodata', no_data_value])
            cmd.extend(['-dstnodata', no_data_value])

        # Now add the filenames
        cmd.extend([source_file, output_file])

        cmd = ' '.join(cmd)
        logger.info("Warping %s with %s" % (source_file, cmd))

        output = utilities.execute_cmd(cmd)
        if len(output) > 0:
            logger.info(output)

    except Exception:
        raise

    finally:
        # Remove the environment variable we set above
        del os.environ['GDAL_PAM_ENABLED']

def ftp_to_remote_location(username, pw, localfile, host, remotefile):
    '''
    Author: David Hill

    Date: 12/5/13

    Description:
      Transfers files from the local machine to a remote location using ftplib.

    Parameters:
      username = Username for ftp account
      pw = Password for ftp account
      host = The ftp server host
      remotefile = Full path of where to store the file
                   (Directories must exist)
      localfile = Full path of file to transfer out

    Returns: None

    Errors: Raises Exception() in the event of error
    '''

    logger = EspaLogging.get_logger(settings.PROCESSING_LOGGER)

    # Make sure the src_file is absolute, otherwise ftp will choke
    if not remotefile.startswith('/'):
        remotefile = ''.join(['/', remotefile])

    pw = urllib2.unquote(pw)

    logger.info("Transferring file from %s to %s"
                % (localfile, 'ftp://%s/%s' % (host, remotefile)))

    ftp = None

    try:
        ftp = ftplib.FTP(host, user=username, passwd=pw, timeout=60)
        with open(localfile, 'rb') as tmp_fd:
            ftp.storbinary(' '.join(['STOR', remotefile]), tmp_fd, 1024)
    finally:
        if ftp:
            ftp.quit()

    logger.info("Transfer complete - FTP")

def get_sleep_duration(start_time):
    """Logs details and returns number of seconds to sleep
    """

    logger = EspaLogging.get_logger(settings.PROCESSING_LOGGER)

    # Determine if we need to sleep
    end_time = datetime.datetime.now()
    seconds_elapsed = (end_time - start_time).seconds
    logger.info('Processing Time Elapsed {0} Seconds'.format(seconds_elapsed))

    seconds_to_sleep = 1
    if seconds_elapsed < settings.MIN_REQUEST_DURATION_IN_SECONDS:
        # Joe-Developer doesn't want to wait so check and skip
        # This directory will not exist for HADOOP processing
        if not os.path.isdir('unittests'):
            seconds_to_sleep = (settings.MIN_REQUEST_DURATION_IN_SECONDS -
                                seconds_elapsed)

    logger.info('Sleeping An Additional {0} Seconds'.format(seconds_to_sleep))

    return seconds_to_sleep

def remote_copy_file_to_file(source_host, source_file, destination_file):
    '''
    Description:
      Use unix 'cp' to copy a file from one place to another on a remote
      machine using ssh.
    '''

    logger = EspaLogging.get_logger(settings.PROCESSING_LOGGER)

    cmd = ' '.join(['ssh', '-q', '-o', 'StrictHostKeyChecking=no',
                    source_host, 'cp', source_file, destination_file])

    # Transfer the data and raise any errors
    output = ''
    try:
        output = utilities.execute_cmd(cmd)
    except Exception as e:
        logger.error("Failed to copy file")
        raise e
    finally:
        if len(output) > 0:
            logger.info(output)

    logger.info("Transfer complete - SSH-CP")

def copy_files_to_directory(source_files, destination_directory):
    '''
    Description:
      Use unix 'cp' to copy files from one place to another on the localhost.
    '''

    logger = EspaLogging.get_logger(settings.PROCESSING_LOGGER)

    if type(source_files) == list:
        for source_file in source_files:
            cmd = ' '.join(['cp', source_file, destination_directory])

            # Transfer the data and raise any errors
            output = ''
            try:
                output = utilities.execute_cmd(cmd)
            except Exception as e:
                logger.error("Failed to copy file")
                raise e
            finally:
                if len(output) > 0:
                    logger.info(output)

    logger.info("Transfer complete - CP")

def move_files_to_directory(source_files, destination_directory):
    '''
    Description:
      Move files from one place to another on the localhost.
    '''

    logger = EspaLogging.get_logger(settings.PROCESSING_LOGGER)

    if type(source_files) == str:
        filename = os.path.basename(source_files)

        new_name = os.path.join(destination_directory, filename)

        os.rename(source_files, new_name)

    elif type(source_files) == list:
        for source_file in source_files:
            filename = os.path.basename(source_file)

            new_name = os.path.join(destination_directory, filename)

            os.rename(source_file, new_name)

    logger.info("Transfer complete - MOVE")