Python WhfLog.debug方法代码示例

# 需要导入模块: import WhfLog [as 别名]
# 或者: from WhfLog import debug [as 别名]
def forecastExists(dir, issueTime, fcstHour):
    """ Check if forecast exists

    Parameters
    ----------
    dir : str
       Full path to the issue time directories
    issueTime : datetime
       The issue time (y,m,d,h)       
    fcstHour:  int
       should be 0 or 3

    Returns
    -------
    bool
    True if the forecast does exist on disk
    """
           
    path = dir + "/"
    path += issueTime.strftime("%Y%m%d%H")
    if (os.path.isdir(path)):
        validTime = issueTime + datetime.timedelta(hours=fcstHour)
        fname = validTime.strftime("%Y%m%d%H%M") + ".LDASIN_DOMAIN1.nc"
        names = df.getFileNames(path)
        for n in names:
            if (n == fname):
                WhfLog.debug("Found %s in %s",  fname, path)
                return True
    return False

# 需要导入模块: import WhfLog [as 别名]
# 或者: from WhfLog import debug [as 别名]
    def _forecastExists(self, dir):
        """ Check if forecast indicated by local state exists

        Parameters
        ----------
        dir : str
           Full path to the issue time directories

        Returns
        -------
        bool
           True if the forecast does exist on disk
        """
           
        path = dir + "/"
        path += self._issue.strftime("%Y%m%d%H")
        if (os.path.isdir(path)):
            fname = self._valid.strftime("%Y%m%d%H%M") + ".LDASIN_DOMAIN1.nc"
            names = df.getFileNames(path)
            for n in names:
                if (n == fname):
                    WhfLog.debug("Found %s in %s",  fname, path)
                    return True
            return False
        else:
            return False

# 需要导入模块: import WhfLog [as 别名]
# 或者: from WhfLog import debug [as 别名]
def run(fileType, configFile, realtime):
   """ Run the script, process any new data
   Parameters
   ----------
   fileType: str
      'HRRR', ...
   configFile : str
      Name of the file with settings
   realtime : boolean
      True if this is realtime
   Returns
   -------
   1 for error, 0 for success
   """   
   good = False
   regriddable = ['HRRR', 'RAP', 'MRMS', 'GFS']
   if (fileType not in regriddable):
      print 'ERROR unknown file type command arg ', fileType
      return 1

   # User must pass the config file into the main driver.
   if not os.path.exists(configFile):
      print 'ERROR forcing engine config file not found:', configFile
      return 1

   # read in fixed main params
   parms = parmRead(configFile, fileType, realtime)

   #if there is not a state file, create one now using newest
   if (not os.path.exists(parms._stateFile)):
      parms.debugPrint()
      createStateFile(parms, fileType, realtime)
        
   # read in state
   state = State(parms._stateFile, fileType)

      # query each directory and get newest model run file for each, then
   # get all for that and previous issue time
   data = df.DataFiles(parms._dataDir, parms._maxFcstHour, fileType)
   data.setNewestFiles(parms._hoursBack)

   # Update the state to reflect changes, returning those files to regrid
   # Regrid 'em
   toProcess = state.lookForNew(data, parms._hoursBack, fileType)
   for f in toProcess:
      try:
         regrid(f, fileType, configFile);
      except:
         WhfLog.error("Could not regrid/downscale %s", f)
      else:
         WhfLog.debug("Adding new file %s, and writing state file", f)
         if (not state.addFileIfNew(f)):
            WhfLog.error("File %s was not new", f)
         else:
            state.write(parms._stateFile, fileType);
          
   # write out state (in case it has not been written yet) and exit
   #state.debugPrint()
   state.write(parms._stateFile, fileType)
   return 0

# 需要导入模块: import WhfLog [as 别名]
# 或者: from WhfLog import debug [as 别名]
   def _allDataFiles(self):
      """Return all data files on disk, in order

      Parameters
      ----------
      none

      Returns
      -------
      list[DataFile]
         The DataFile specs, in order oldest to newest

      """

      # get the yyyymmdd subdirs
      dirs = getYyyymmddSubdirectories(self._topDir)

      # sort them into ascending order
      dirs = sorted(dirs)

      if not dirs:
         # nothing there
         WhfLog.debug("_allDataFiles: No data in %s", self._topDir)
         return []
      else:
         # make one big array
         ret = []
         for d in dirs:
            ret.extend(self._allDataFilesInDir(d))
         return ret

# 需要导入模块: import WhfLog [as 别名]
# 或者: from WhfLog import debug [as 别名]
    def layerIfReady(self, parms, configFile):
        """  Perform layering if state is such that it should be done

        Parameters
        ----------
        parms : Parms
           Parameters
        configFile : string
           name of file with settings

        Returns
        -------
        bool
           True if layering was done, or had previously been done
        """
        
        if (self._layered):
            return True
        if ((not self._hrrr) and (not self._rap)):
            return False
        if (self._hrrr and self._rap):
            self._layer(parms, configFile)
            self._layered = True
            return True
        if (self._rap and not self._hrrr):
            ntime = datetime.datetime.utcnow()
            diff = ntime - self._clockTime
            idiff = diff.total_seconds()
            if (idiff > parms._maxWaitSeconds):
                WhfLog.debug("timeout..Should layer, dt=%d", idiff)
                self._passthroughRap(parms)
                self._layered = True
                return True
        return False

# 需要导入模块: import WhfLog [as 别名]
# 或者: from WhfLog import debug [as 别名]
def filterWithinNHours(files, type, ftime, N):
   """Filter a list of file names with assumed format to those in a time range
       
   Parameters
   ----------
   files: list[str]
      file names, each with parent directory: 'yyyymmdd/<file>
   type: str
      file type string, 'RAP', 'HRRR', 'MRMS', GFS'
   ftime: ForecastTime
      time to compare against, assumed most recent time
   N: int
      Number of hours back from ftime to keep

   Returns
   -------
   list[str]
      subset of input files with issue time is in the range [ftime-N,ftime]

   """

   ret = []
   for f in files:
      df = DataFile(f[0:8], f[9:], type)
      if (df._ok):
         ithFtime = df._time
         if (ithFtime.withinNHours(ftime, N)):
            ret.append(f)
         else:
            WhfLog.debug("Did not append file, too old compared to %s  file=%s", ftime.debugString(), f)

   WhfLog.debug("filtering within %d hours, input length %d output %d",
                 N, len(files), len(ret))
   return ret

# 需要导入模块: import WhfLog [as 别名]
# 或者: from WhfLog import debug [as 别名]
   def lookForNew(self, data, hoursBack, fileType):
      """ See if new data has arrived compared to state.
      If a new issue time, purge older stuff from state.

      Parameters
      ----------
      data: DataFiles
         The newest data
      hoursBack: int
         Maximum number of hours back to keep data in state
      fileType : str
         'HRRR', 'RAP', ...
      Returns
      -------
      list[str]
          The data file names that are are to be added to state
      """
         
      ret = []
      fnames = data.getFnames()
      if (not fnames):
         return ret

      if (self.isEmpty()):
         WhfLog.debug("Adding to empty list")
      else:
         sname = self.newest()
         if (not sname):
            WhfLog.error("Expected file, got none")
            return ret
         self._analyzeNewest(fnames[-1], sname, hoursBack, fileType)
      for f in fnames:
         if (self._isNew(f)):
            ret.append(f)
      return ret

# 需要导入模块: import WhfLog [as 别名]
# 或者: from WhfLog import debug [as 别名]
def obsExists(dir, issueTime):
    """ Check if obs exists

    Parameters
    ----------
    dir : str
       Full path to the MRMS directories
    issueTime : datetime
       The issue time (y,m,d,h)       

    Returns
    -------
    bool
    True if the data does exist on disk
    """
           
    path = dir + "/"
    path += issueTime.strftime("%Y%m%d%H")
    if (os.path.isdir(path)):
        fname = issueTime.strftime("%Y%m%d%H%M") + ".LDASIN_DOMAIN1.nc"
        names = df.getFileNames(path)
        for n in names:
            if (n == fname):
                WhfLog.debug("Found %s in %s",  fname, path)
                return True
    return False

# 需要导入模块: import WhfLog [as 别名]
# 或者: from WhfLog import debug [as 别名]
 def debugPrint(self):
     """ logging debug of content
     """
     WhfLog.debug("FcstStep: empty=%d", self._empty)
     if (self._empty):
         return
     WhfLog.debug("FcstStep[%d] hrrr0:%d hrrr3:%d rap0:%d rap3:%d mrms:%d lay:%d",
                   self._step, self._hrrr0, self._hrrr3, self._rap0,
                   self._rap3, self._mrms, self._layered)

# 需要导入模块: import WhfLog [as 别名]
# 或者: from WhfLog import debug [as 别名]
 def debugPrint(self):
     """ WhfLog debug of content
     """
     WhfLog.debug("Model: empty=%d", df.boolToInt(self._empty))
     if (self._empty):
         return
     WhfLog.debug("Model: Issue=%s  clockTime=%s",
                   self._issue.strftime("%Y%m%d%H"),
                   self._clockTime.strftime("%Y-%m-%d_%H:%M:%S"))

# 需要导入模块: import WhfLog [as 别名]
# 或者: from WhfLog import debug [as 别名]
def main(argv):

    # User must pass the config file into the main driver.
    configFile = argv[0]
    if not os.path.exists(configFile):
        print 'ERROR forcing engine config file not found.'
        return 1
    # read in fixed main params
    parms = parmRead(configFile)

    # query each directory to get newest thing, and update overall newest
    #WhfLog.debug("Looking in %s and %s", parms._hrrrDir, parms._rapDir)
    newestT = df.newestIssueTime(parms._hrrrDir)
    newestT2 = df.newestIssueTime(parms._rapDir)
    if (not newestT) and (not newestT2):
        WhfLog.debug("NO INPUT DATA available")
        return 0
    
    if (newestT2 > newestT):
        newestT = newestT2
    #WhfLog.debug("Newest issue time = %s", newestT)

    # if there is not a state file, create one now using newest
    if (not os.path.exists(parms._stateFile)):
        state = State()
        WhfLog.info("Initializing")
        state.initialSetup(parms)
        state.initialize(parms, newestT)
        state.write(parms._stateFile)

    # Normal processing situation
    #WhfLog.debug("Look for Layering....")
    
    # read in state
    state2 = State()
    state2.initFromStateFile(parms._stateFile)
    if state2.isEmpty():
        # error return here
        return 0
    
    # check for new issue time
    if (state2.isNewModelIssueTime(newestT)):
        WhfLog.info("Re-Initializing state, new model issue time %s", newestT)
        state2.initialize(parms, newestT)

    # update availability
    state2.setCurrentModelAvailability(parms)

    # layer if appropriate
    state2.layerIfReady(parms, configFile)

    # write out final state
    state2.write(parms._stateFile)
    return 0

# 需要导入模块: import WhfLog [as 别名]
# 或者: from WhfLog import debug [as 别名]
   def updateWithNew(self, data, hoursBack):
      """ Update internal state with new data

      The dataType is used to determine which part of state to update

      Parameters
      ----------
      data: DataFiles
         The newest data
      hoursBack: int
         Maximum number of hours back to keep data in state

      Returns
      -------
      list[str]
          The data file names that are are newly added to state
      """
         
      ret = []
      fnames = data.getFnames()
      if (not fnames):
         return ret

      if (self.isEmpty()):
         WhfLog.debug("Adding to empty %s list")
      else:
         sname = self.newest()
         if (not sname):
            WhfLog.error("Expected file, got none")
            return ret
         if (fnames[-1] > sname):
            WhfLog.debug("Newer time encountered")
            # see if issue time has increased and if so, purge old stuff
            # create DataFile objects
            try:
               df0 = df.DataFile(sname[0:8], sname[9:], 'CFS')
               df1 = df.DataFile(fnames[-1][0:8], fnames[-1][9:], 'CFS')
            except FilenameMatchError as fe:
               WhfLog.debug("Skipping file use due to %s", fe)
            except InvalidArgumentError as ie:
               WhfLog.debug("Skipping file use due to %s", ie)

            if (df0._time.inputIsNewerIssueHour(df1._time)):
               WhfLog.debug("Issue hour has increased, purge now")
               self.update(df1._time, hoursBack)

      for f in fnames:
         if (self.addFileIfNew(f)):
            ret.append(f)

      self.sortFiles()
      return ret

# 需要导入模块: import WhfLog [as 别名]
# 或者: from WhfLog import debug [as 别名]
 def debugPrint(self):
    """ Debug logging of content
    """
    WhfLog.debug("Parms: CFS_data = %s", self._cfsDir)
    WhfLog.debug("Parms: CFS_num_ensembles = %d", self._cfsNumEnsemble)
    WhfLog.debug("Parms: MaxFcstHourCfs = %d", self._maxFcstHourCfs)
    WhfLog.debug("Parms: StateFile = %s", self._stateFile)

# 需要导入模块: import WhfLog [as 别名]
# 或者: from WhfLog import debug [as 别名]
   def _analyzeNewest(self, dataNewest, stateNewest, hoursBack, fileType):
      if (dataNewest <= stateNewest):
         return

      # see if issue time has increased and if so, purge old stuff
      # create DataFile objects, which requires breaking the full
      # file into yymmdd/filename
      sind = stateNewest.find('/')
      if (sind < 0):
         raise FileNameMatchError('Cannot parse directory from ' + stateNewest)
      nind = dataNewest.find('/')
      if (nind < 0):
         raise FileNameMatchError('Cannot parse directory from ' + dataNewest)
      symd = stateNewest[:sind]
      sfile = stateNewest[sind+1:]
      nymd = dataNewest[:nind]
      nfile = dataNewest[nind+1:]
      WhfLog.debug("Checking %s / %s  against %s / %s", symd, sfile, nymd, nfile)
      try:
         df0 = df.DataFile(symd, sfile, fileType)
         df1 = df.DataFile(nymd, nfile, fileType)
      except FilenameMatchError as fe:
         WhfLog.debug("Cannot update due to %s", fe)
      except InvalidArgumentError as ie:
         WhfLog.debug("Cannot update due to %s", ie)

      if (df0._time.inputIsNewerIssueHour(df1._time)):
         WhfLog.debug("%s Issue hour has increased, purge now", fileType)
         self._update(df1._time, hoursBack, fileType)

# 需要导入模块: import WhfLog [as 别名]
# 或者: from WhfLog import debug [as 别名]
 def debugPrint(self):
    """ Debug logging of content
    """
    WhfLog.debug("%s data = %s", self._dataType, self._dataDir)
    WhfLog.debug("%s source = %s", self._dataType, self._sourceDataDir)
    WhfLog.debug("%s statefile = %s", self._dataType, self._stateFile)
    WhfLog.debug("%s format = %s", self._dataType, self._format)