Python Log.d方法代码示例

# 需要导入模块: from virtualisation.misc.log import Log [as 别名]
# 或者: from virtualisation.misc.log.Log import d [as 别名]
 def deleteGraph(self, graphName):
     queryString = "DEFINE sql:log-enable 3 DROP SILENT GRAPH <" + self.getGraphURI(graphName) + ">"
     L.d("deleteGraph using query:", queryString)
     sparql = self.getSparqlObject(graphName, queryString)
     sparql.setTimeout(300)
     try:
         ret = sparql.query()
         return True
     except Exception as e:
         L.e("Error in deleteGraph:", e.message)
         return False

# 需要导入模块: from virtualisation.misc.log import Log [as 别名]
# 或者: from virtualisation.misc.log.Log import d [as 别名]
 def checkTimeRelevantMetrics(self, lastUpdate):
     L.d("ReputationSystem: checkTimeRelevantMetrics called for Stream", self.description.fullSensorID)
     L.d("ReputationSystem:", lastUpdate, self.timestamp)
     if (lastUpdate is not None) and (lastUpdate == self.timestamp):  # check if there was an update in the meanwhile
         L.d("ReputationSystem: There was no update, lets punish!")
         qoiValues = {}
         for metric in self.metrics:
             value = metric.nonValueUpdate()
             if value:
                 qoiValues[value[0]] = value[1]
             
         self.avgQoIManager.calculateAvgQualities(qoiValues)
     self.addClockJob()

# 需要导入模块: from virtualisation.misc.log import Log [as 别名]
# 或者: from virtualisation.misc.log.Log import d [as 别名]
    def get_observations(self, uuid, start=None, end=None, format='json', onlyLast=False, fields=None, offset=0):
        from virtualisation.resourcemanagement.resourcemanagement import ResourceManagement

        w = self.rm.getWrapperByUUID(uuid)
        if not w:
            return None

        sd = w.getSensorDescription()

        # prepare query
        _filter = ["sensor_uuid = '%s'" % uuid]
        order = "ORDER BY sampling_time"
        limitation = ""
        if onlyLast:
            order += " DESC"
        else:
            if start:
                _filter.append("sampling_time >= TIMESTAMP '%s'" % start)
            if end:
                _filter.append("sampling_time <= TIMESTAMP '%s'" % end)
        _filter = "WHERE " + " and ".join(_filter)

        if fields:
            fields = fields.split(',')
            fields_ = []
            for ft in fields:
                fields_.append("data->'%s' AS %s" % (ft, ft))
            fields_.append("quality")
        else:
            fields_ = SQL.cp_observation_fields
        limitation = "LIMIT %d" % (1 if onlyLast else self.PAGINATION_LIMIT)
        query = "SELECT %s FROM %s.cp_observations %s %s %s OFFSET %d;" % (",".join(fields_), SQL.SCHEMA, _filter, order, limitation, offset)
        # query = "SELECT %s FROM %s.cp_observations %s %s;" % (",".join(fields_), SQL.SCHEMA, _filter, order)

        L.d("SQL: executing query", query)

        try:
            # need a new cursor object to no interfere with the state of the class's inserting cursor
            cursor = self.conn.cursor()
            cursor.execute(query)
            data = cursor.fetchall()
            data2 = [list(x) for x in data]
            del data
            if format in ('n3', 'nt', 'xml', 'turtle', 'pretty-xml', 'trix'):
                if ResourceManagement.args.messagebus or ResourceManagement.args.triplestore:
                    if fields:
                        observations = []
                        qualities = []
                        for x in data2:
                            tmp = JOb()
                            for i in range(0, len(fields)):
                                ft = fields[i]
                                tmp[ft] = JOb(x[i])
                            tmp.fields = fields
                            observations.append(tmp)
                            qualities.append(JOb(x[-1]))
                    else:
                        observations = [JOb(x[3]) for x in data2]
                        qualities = [JOb(x[4]) for x in data2]
                    g = self.rm.annotator.annotateObservation(observations, sd, None, qualities)
                    del observations
                    del qualities
                    del query
                    return g.serialize(format=format)
                else:
                    return "Error: requires messagebus or triplestore to be enabled"
            else:
                # search in all columns in each row for a datetime.datetime and parse it
                for i in range(0, len(data2)):
                    data2[i] = map(lambda x: x.strftime("%Y-%m-%d %H:%M:%S") if isinstance(x, datetime.datetime) else x, data2[i])
                json_list = []
                for x in data2:
                    if fields:
                        # y = JOb({})
                        y = {}
                        for i in range(0, len(fields)):
                            # ft = fields[i]
                            # y[ft] = JOb(x[i])
                            y[fields[i]] = x[i]
                        # y.quality = JOb(x[-1])
                        # y.fields = fields
                        y["fields"] = fields
                        y["quality"] = x[-1]

                    else:
                        # y = JOb(x[3])
                        # y.quality = JOb(x[4])
                        y = x[3]
                        y["quality"] = x[4]
                    json_list.append(y)
                del query
                del data2
                # return JOb(json_list).dumps()
                return json_list

        except Exception as e:
            L.e("SQL:", e)
            L.e("SQL query used:", query)
            return "Error: " + str(e)

# 需要导入模块: from virtualisation.misc.log import Log [as 别名]
# 或者: from virtualisation.misc.log.Log import d [as 别名]
    def update(self, data):
        self.updatecounter += 1

        # special case when no fields are in data
        # (fault recovery is not ready yet)
        if len(data.fields) == 0:
            self.rewardAndPunishment.update(False)
            self.absoluteValue = float("inf")
            self.ratedValue = self.rewardAndPunishment.value()
            return


        wrongFieldList = []
        for field in data.fields:
            if field not in data:
                wrongFieldList.append(field)
                continue

            dataTypeStr = self.repsys.description.field[field].dataType
            dataType = utils.getType(dataTypeStr)
            minValue, maxValue = self.getMinMaxValue(field, data)

            value = data[field].value
            # print "field:", field, "value:", value, "min:", minValue, "max:", maxValue, "dataType:", dataTypeStr, dataType, "value type:", type(value)

            if minValue and maxValue:
                if dataTypeStr == "datetime.datetime":
                    minValue = datetime.datetime.strptime(minValue, AbstractClock.parserformat)
                    maxValue = datetime.datetime.strptime(maxValue, AbstractClock.parserformat)
                else:
                    maxValue = dataType(maxValue)
                    minValue = dataType(minValue)

            # everything might be a string => first check for type, then try to cast, afterwards check min and max
            wrongValue = False
            if not isinstance(value, dataType): # type(value) is not dataType:
                try:
                    # special handling for datetime as format is needed
                    if dataTypeStr == "datetime.datetime":
                        value = datetime.datetime.strptime(value, self.repsys.description.field[field].format)
                    else:
                        value = dataType(value)
                except ValueError:
                    wrongFieldList.append(field)
                    wrongValue = True

            if not wrongValue:
                # now check if value is within min max interval
                if minValue and minValue is not "":
                    if value < minValue:
                        wrongFieldList.append(field)
                elif maxValue and maxValue is not "":
                    if value > maxValue:
                        wrongFieldList.append(field)
# 			print "Correctness for", self.repsys.description.fullSensorID, len(wrongFieldList), value, minValue, maxValue

        nrWrongFields = len(wrongFieldList)
        if nrWrongFields > 0:
            L.d("Correctness wrong fields:", nrWrongFields, "(", ",".join(wrongFieldList), ")")

        if data.recovered or (nrWrongFields >= 1):
            self.rewardAndPunishment.update(False)
        else:
            self.rewardAndPunishment.update(True)

        self.ratedValue = self.rewardAndPunishment.value()
        self.absoluteValue = 1 - nrWrongFields / len(data.fields)
        self.min = min(self.min, self.absoluteValue)
        self.mean = ((self.updatecounter - 1) * self.mean) / self.updatecounter + float(
            self.absoluteValue) / self.updatecounter

        correctness = JSONObject()
        correctness.wrongFields = wrongFieldList
        correctness.absoluteValue = self.absoluteValue
        correctness.ratedValue = self.ratedValue
        correctness.unit = self.unit
        # 		print "correctness:", self.ratedValue, self.absoluteValue
        return (self.name, correctness)

# 需要导入模块: from virtualisation.misc.log import Log [as 别名]
# 或者: from virtualisation.misc.log.Log import d [as 别名]
    def update(self, data):
        # special case when no fields are in data
        # (fault recovery is not ready yet)
        if len(data.fields) == 0:
            self.rewardAndPunishment.update(False)
            self.absoluteValue = float("inf")
            self.ratedValue = self.rewardAndPunishment.value()
            return

        # look for expected fields in sensor description, look only for non optional fields
        fields = self.repsys.description.fields
        fields = [x for x in fields if not self.repsys.description.field[x].optional]
        
        receivedFields = data.fields

        # check if expected and received identical, how to handle received fields with no values?
        nrOfMissingFields = 0
        missingFields = set()
        if set(fields).difference(set(receivedFields)):
            # lists are different
            missingFields = set(fields).difference(set(receivedFields))
            nrOfMissingFields = len(missingFields)

        # now go through all fields and check for NULL, NA,...
        nrOfWrongFields = 0
        wrongFields = set()
        wrongValues = ['None', 'Null', '', 'NA']  #TODO make the list of wrong values configurable
        for field in data.fields:
            if field in data:
                value = data[field].value
                if value is None or value in wrongValues:
                    nrOfWrongFields += 1
                    wrongFields.add(field)
            else:
                nrOfWrongFields += 1
                wrongFields.add(field)

        if nrOfMissingFields > 0:
            L.d("Completeness missing fields:", nrOfMissingFields, "(", ",".join(missingFields), ")")
        if nrOfWrongFields > 0:
            L.d("Completeness wrong fields:", nrOfWrongFields, "(", ",".join(wrongFields), ")")

        length = len(self.repsys.description.fields)
        currentLength = length - nrOfMissingFields - nrOfWrongFields
        self.updatecounter += 1
        if not self.goal:
            self.goal = length
            self.min = float(length)
            self.mean = float(length)
        # 			return (length, self.rewardAndPunishment.value())
        else:
            self.min = min(self.min, currentLength)
            self.mean = ((self.updatecounter - 1) * self.mean) / self.updatecounter + float(
                currentLength) / self.updatecounter
                
        if data.recovered:
            self.rewardAndPunishment.update(False)
        else:
            self.rewardAndPunishment.update(self.goal == currentLength)
        self.absoluteValue = currentLength
        self.ratedValue = self.rewardAndPunishment.value()

        completeness = JSONObject()
        completeness.missingFields = list(missingFields | wrongFields)
        completeness.absoluteValue = self.absoluteValue
        completeness.ratedValue = self.ratedValue
        completeness.unit = self.unit


        # 		print completeness.dumps()

        # 		print "completeness:", self.name, completeness
        # 		print (self.name, missingFields)
        return (self.name, completeness)

# 需要导入模块: from virtualisation.misc.log import Log [as 别名]
# 或者: from virtualisation.misc.log.Log import d [as 别名]
    def update(self):
        from virtualisation.resourcemanagement.resourcemanagement import ResourceManagement

        # print "time", self.clock.now()
        latStart = datetime.now()
        L.d("processing:", self.getSensorDescription().sensorID)
        # L.d(self.clock.now())
        if self.replaymode:
            self.stats.startMeasurement("Update_replay")
            #             self.clock.pause()
            if self.historyreader:
                L.d2("abstractwrapper get data")
                self.stats.startMeasurement("Update_replay.Historyreader")
                data_raw = self.historyreader.tick(self.clock)
                self.stats.stopMeasurement("Update_replay.Historyreader")
                L.d2("abstractwrapper received data:", str(data_raw))
                if data_raw:
                    data_list = [data_raw] if not self.historyreader.multiple_observations else data_raw
                    for data in data_list:
                        try:
                            L.d2("abstractwrapper parse data")
                            # print "data to parse", data
                            self.stats.startMeasurement("Update_replay.Historyparser")
                            parsed = self.historyparser.parse(data, self.clock)
                            self.stats.stopMeasurement("Update_replay.Historyparser")
                            L.d2("abstractwrapper parsed data:", str(parsed))
                            del data
                            if parsed:
                                self.stats.startMeasurement("Update_replay.Preparation")
                                ObservationIDGenerator.addObservationIDToFields(parsed)
                                parsed.producedInReplayMode = True
                                parsed.recovered = False
                                parsed.latency = (datetime.now() - latStart).total_seconds()
                                self.stats.stopMeasurement("Update_replay.Preparation")

                                # QoI Start
                                quality = None
                                if self.qoiSystem:
                                    L.d2("abstractwrapper get quality")
                                    self.stats.startMeasurement("Update_replay.Quality")
                                    quality = self.qoiSystem.addData(self.getSensorDescription(), parsed, self.clock)
                                    self.stats.stopMeasurement("Update_replay.Quality")
                                    L.d2("abstractwrapper quality:", quality)
                                if self.faultRecoveryActive:
                                    L.d2("abstractwrapper update fault recovery")
                                    self.stats.startMeasurement("Update_replay.FaultRecoveryUpdate")
                                    self.updateFaultRecoveries(parsed, quality)
                                    self.stats.stopMeasurement("Update_replay.FaultRecoveryUpdate")
                                    L.d2("abstractwrapper fault recovery updated")

                                self.stats.startMeasurement("Update_replay.Receiver")
                                for r in self.receiver:
                                    L.d2("abstractwrapper start receiver", r)
                                    r.receive(parsed, self.getSensorDescription(), self.clock, quality)
                                    L.d2("abstractwrapper receiver", r, "finished")
                                self.stats.stopMeasurement("Update_replay.Receiver")
                        except Exception as e:
                            L.e("Error while updating sensor", self.getSensorDescription().fullSensorID, e)
                        finally:
                            if ResourceManagement.args.gentle:
                                self.clock.sleep()
                else:
                    L.d("there is no data, ask fault recovery1")
                    # L.i(self.getSensorDescription().sensorID)
                    # L.i(self.clock.now())
                    try:
                        self.stats.startMeasurement("Update_replay.Recovery")
                        data = JSONObject()
                        data.latency = 0
                        data.producedInReplayMode = True
                        data.recovered = True

                        data.fields = []
                        for n in self.getSensorDescription().fields:
                            if n in self.faultRecoveries and self.faultRecoveries[n].isReady():
                                data.fields.append(n)
                                data[n] = JSONObject()
                                # at this point the dataType is in FAULT_RECOVERY_SUPPORTED_DATATYPES and we can safely use cast
                                data[n].value = self.faultRecoveryCast(
                                    self.faultRecoveries[n].getEstimation(),
                                    self.getSensorDescription().field[n].dataType,
                                )
                                data[n].propertyName = self.getSensorDescription().field[n].propertyName
                                data[n].propertyURI = self.getSensorDescription().field[n].propertyURI
                                if "unit" in self.getSensorDescription().field[n]:
                                    data[n].unit = self.getSensorDescription().field[n].unit
                                data[n].sensorID = self.getSensorDescription().fullSensorID
                                data[n].observationSamplingTime = self.clock.timeAsString()
                                data[n].observationResultTime = data[n].observationSamplingTime
                        self.stats.stopMeasurement("Update_replay.Recovery")

                        self.stats.startMeasurement("Update_replay.ObservationIDGenerator")
                        ObservationIDGenerator.addObservationIDToFields(data)
                        self.stats.stopMeasurement("Update_replay.ObservationIDGenerator")

                        quality = None
                        if self.qoiSystem:
                            self.stats.startMeasurement("Update_replay.Quality")
                            quality = self.qoiSystem.addData(self.getSensorDescription(), data, self.clock)
                            self.stats.stopMeasurement("Update_replay.Quality")
#.........这里部分代码省略.........