Python json.loads方法代码示例

# 需要导入模块: import json [as 别名]
# 或者: from json import loads [as 别名]
def _initialize_members(self, db_name, hdfs_app_path, kafka_consumer, conf_type):

        # get logger instance.
        self._logger = Util.get_logger('SPOT.INGEST.WRK.FLOW')

        self._db_name = db_name
        self._hdfs_app_path = hdfs_app_path

        # read proxy configuration.
        self._script_path = os.path.dirname(os.path.abspath(__file__))
        conf_file = "{0}/ingest_conf.json".format(os.path.dirname(os.path.dirname(self._script_path)))
        conf = json.loads(open(conf_file).read())
        self._conf = conf["pipelines"][conf_type]
        self._id = "spot-{0}-worker".format(conf_type)

        self._process_opt = self._conf['process_opt']
        self._local_staging = self._conf['local_staging']
        self.kafka_consumer = kafka_consumer

        # self._cursor = hive_engine.create_connection()
        self._cursor = hive_engine 

# 需要导入模块: import json [as 别名]
# 或者: from json import loads [as 别名]
def _initialize_members(self,db_name,hdfs_app_path,kafka_consumer,conf_type,processes):
        
        # get logger instance.
        self._logger = Util.get_logger('SPOT.INGEST.WRK.PROXY')

        self._db_name = db_name
        self._hdfs_app_path = hdfs_app_path
        self._kafka_consumer = kafka_consumer

        # read proxy configuration.
        self._script_path = os.path.dirname(os.path.abspath(__file__))
        conf_file = "{0}/ingest_conf.json".format(os.path.dirname(os.path.dirname(self._script_path)))
        conf = json.loads(open(conf_file).read())
        self._spark_conf  = conf["spark-streaming"]
        self._conf = conf["pipelines"][conf_type]
        self._processes = processes 

# 需要导入模块: import json [as 别名]
# 或者: from json import loads [as 别名]
def _get_suspicious_details(self):
        uri_list = []
        iana_conf_file = "{0}/components/iana/iana_config.json".format(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
        if os.path.isfile(iana_conf_file):
            iana_config  = json.loads(open(iana_conf_file).read())
            proxy_iana = IanaTransform(iana_config["IANA"])

        for conn in self._proxy_scores:
            clientip = conn[self._conf["proxy_score_fields"]["clientip"]]
            fulluri = conn[self._conf["proxy_score_fields"]["fulluri"]]
            date=conn[self._conf["proxy_score_fields"]["p_date"]].split('-')
            if len(date) == 3:
                year=date[0]
                month=date[1].zfill(2)
                day=date[2].zfill(2)
                hh=(conn[self._conf["proxy_score_fields"]["p_time"]].split(":"))[0]
                self._get_proxy_details(fulluri,clientip,year,month,day,hh,proxy_iana) 

# 需要导入模块: import json [as 别名]
# 或者: from json import loads [as 别名]
def _get_suspicious_details(self):

        iana_conf_file = "{0}/components/iana/iana_config.json".format(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
        if os.path.isfile(iana_conf_file):
            iana_config  = json.loads(open(iana_conf_file).read())
            dns_iana = IanaTransform(iana_config["IANA"])
        
        for conn in self._dns_scores:

            timestamp = conn[self._conf["dns_score_fields"]["unix_tstamp"]]
            full_date = datetime.datetime.utcfromtimestamp(int(timestamp)).strftime('%Y-%m-%d %H:%M:%S')

            date = full_date.split(" ")[0].split("-")
            # get date parameters.
            yr = date[0]
            mn = date[1]
            dy = date[2]
            time = full_date.split(" ")[1].split(":")
            hh = int(time[0])

            dns_qry_name = conn[self._conf["dns_score_fields"]["dns_qry_name"]]
            self._get_dns_details(dns_qry_name,yr,mn,dy,hh,dns_iana) 

# 需要导入模块: import json [as 别名]
# 或者: from json import loads [as 别名]
def _call_gti(self, command, num_values):
        try:
            response_json = check_output(command, shell=True)
            result_dict = json.loads(response_json[0:len(response_json) - 1])
            responses = result_dict['a']
            return responses

        except CalledProcessError as e:
            self._logger.error("Error calling McAfee GTI client in gti module: " + e.output)
            error_resp = [{self.REP_KEY: self.DEFAULT_REP}] * num_values
            return error_resp

        except ValueError as e:
            self._logger.error("Error reading JSON response in gti module: " + e.message)
            error_resp = [{self.REP_KEY: self.DEFAULT_REP}] * num_values
            return error_resp 

# 需要导入模块: import json [as 别名]
# 或者: from json import loads [as 别名]
def refresh(self):
        try:
            #open the data url
            self.req = urlopen(self.data_url)

            #read data from the url
            self.raw_data = self.req.read()

            #load in the json
            self.json_data = json.loads(self.raw_data.decode())

            #get time from json
            self.time = datetime.fromtimestamp(self.parser.time(self.json_data))

            #load all the aircarft
            self.aircraft = self.parser.aircraft_data(self.json_data, self.time)

        except Exception:
            print("exception in FlightData.refresh():")
            traceback.print_exc() 

# 需要导入模块: import json [as 别名]
# 或者: from json import loads [as 别名]
def test_Index(self):
        """Test for the index."""
        response_get = self.client.get("/{}".format(self.API_NAME))
        endpoints = json.loads(response_get.data.decode('utf-8'))
        response_post = self.client.post(
            "/{}".format(self.API_NAME), data=dict(foo="bar"))
        response_put = self.client.put(
            "/{}".format(self.API_NAME), data=dict(foo="bar"))
        response_delete = self.client.delete("/{}".format(self.API_NAME))
        assert "@context" in endpoints
        assert endpoints["@id"] == "/{}".format(self.API_NAME)
        assert endpoints["@type"] == "EntryPoint"
        assert response_get.status_code == 200
        assert response_post.status_code == 405
        assert response_put.status_code == 405
        assert response_delete.status_code == 405 

# 需要导入模块: import json [as 别名]
# 或者: from json import loads [as 别名]
def test_Vocab(self):
        """Test the vocab."""
        response_get = self.client.get("/{}/vocab#".format(self.API_NAME))
        response_get_data = json.loads(response_get.data.decode('utf-8'))

        assert "@context" in response_get_data
        assert response_get_data["@type"] == "ApiDocumentation"
        assert response_get_data["@id"] == "{}{}/vocab".format(
            self.HYDRUS_SERVER_URL, self.API_NAME)
        assert response_get.status_code == 200

        response_delete = self.client.delete(
            "/{}/vocab#".format(self.API_NAME))
        assert response_delete.status_code == 405

        response_put = self.client.put(
            "/{}/vocab#".format(self.API_NAME), data=json.dumps(dict(foo='bar')))
        assert response_put.status_code == 405

        response_post = self.client.post(
            "/{}/vocab#".format(self.API_NAME), data=json.dumps(dict(foo='bar')))
        assert response_post.status_code == 405 

# 需要导入模块: import json [as 别名]
# 或者: from json import loads [as 别名]
def test_object_PUT_at_id(self):
        """Create object in collection using PUT at specific ID."""
        index = self.client.get("/{}".format(self.API_NAME))
        assert index.status_code == 200
        endpoints = json.loads(index.data.decode('utf-8'))
        for endpoint in endpoints:
            collection_name = "/".join(endpoints[endpoint].split(
                "/{}/".format(self.API_NAME))[1:])
            if collection_name in self.doc.collections:
                collection = self.doc.collections[collection_name]["collection"]
                class_ = self.doc.parsed_classes[collection.class_.title]["class"]
                class_methods = [x.method for x in class_.supportedOperation]
                dummy_object = gen_dummy_object(
                    collection.class_.title, self.doc)
                if "PUT" in class_methods:
                    dummy_object = gen_dummy_object(
                        collection.class_.title, self.doc)
                    put_response = self.client.put('{}/{}'.format(
                        endpoints[endpoint], uuid.uuid4()), data=json.dumps(dummy_object))
                    assert put_response.status_code == 201 

# 需要导入模块: import json [as 别名]
# 或者: from json import loads [as 别名]
def test_object_PUT_at_ids(self):
        index = self.client.get("/{}".format(self.API_NAME))
        assert index.status_code == 200
        endpoints = json.loads(index.data.decode('utf-8'))
        for endpoint in endpoints:
            collection_name = "/".join(endpoints[endpoint].split(
                "/{}/".format(self.API_NAME))[1:])
            if collection_name in self.doc.collections:
                collection = self.doc.collections[collection_name]["collection"]
                class_ = self.doc.parsed_classes[collection.class_.title]["class"]
                class_methods = [x.method for x in class_.supportedOperation]
                data_ = {"data": list()}
                objects = list()
                ids = ""
                for index in range(3):
                    objects.append(gen_dummy_object(
                        collection.class_.title, self.doc))
                    ids = "{},".format(uuid.uuid4())
                data_["data"] = objects
                if "PUT" in class_methods:
                    put_response = self.client.put(
                        '{}/add/{}'.format(endpoints[endpoint], ids),
                        data=json.dumps(data_))
                    assert put_response.status_code == 201 

# 需要导入模块: import json [as 别名]
# 或者: from json import loads [as 别名]
def test_endpointClass_POST(self):
        """Check non collection Class POST."""
        index = self.client.get("/{}".format(self.API_NAME))
        assert index.status_code == 200
        endpoints = json.loads(index.data.decode('utf-8'))
        for endpoint in endpoints:
            if endpoint not in ["@context", "@id", "@type"]:
                class_name = "/".join(endpoints[endpoint].split(
                    "/{}/".format(self.API_NAME))[1:])
                if class_name not in self.doc.collections:
                    class_ = self.doc.parsed_classes[class_name]["class"]
                    class_methods = [
                        x.method for x in class_.supportedOperation]
                    if "POST" in class_methods:
                        dummy_object = gen_dummy_object(class_.title, self.doc)
                        post_response = self.client.post(
                            endpoints[endpoint], data=json.dumps(dummy_object))
                        assert post_response.status_code == 200 

# 需要导入模块: import json [as 别名]
# 或者: from json import loads [as 别名]
def test_endpointClass_DELETE(self):
        """Check non collection Class DELETE."""
        index = self.client.get("/{}".format(self.API_NAME))
        assert index.status_code == 200
        endpoints = json.loads(index.data.decode('utf-8'))
        for endpoint in endpoints:
            if endpoint not in ["@context", "@id", "@type"]:
                class_name = "/".join(endpoints[endpoint].split(
                    "/{}/".format(self.API_NAME))[1:])
                if class_name not in self.doc.collections:
                    class_ = self.doc.parsed_classes[class_name]["class"]
                    class_methods = [
                        x.method for x in class_.supportedOperation]
                    if "DELETE" in class_methods:
                        delete_response = self.client.delete(
                            endpoints[endpoint])
                        assert delete_response.status_code == 200 

# 需要导入模块: import json [as 别名]
# 或者: from json import loads [as 别名]
def test_endpointClass_GET(self):
        """Check non collection Class GET."""
        index = self.client.get("/{}".format(self.API_NAME))
        assert index.status_code == 200
        endpoints = json.loads(index.data.decode('utf-8'))
        for endpoint in endpoints:
            if endpoint not in ["@context", "@id", "@type"]:
                class_name = "/".join(endpoints[endpoint].split(
                    "/{}/".format(self.API_NAME))[1:])
                if class_name not in self.doc.collections:
                    class_ = self.doc.parsed_classes[class_name]["class"]
                    class_methods = [
                        x.method for x in class_.supportedOperation]
                    if "GET" in class_methods:
                        response_get = self.client.get(endpoints[endpoint])
                        assert response_get.status_code == 200
                        response_get_data = json.loads(
                            response_get.data.decode('utf-8'))
                        assert "@context" in response_get_data
                        assert "@id" in response_get_data
                        assert "@type" in response_get_data 

# 需要导入模块: import json [as 别名]
# 或者: from json import loads [as 别名]
def test_IriTemplate(self):
        """Test structure of IriTemplates attached to collections"""
        index = self.client.get("/{}".format(self.API_NAME))
        assert index.status_code == 200
        endpoints = json.loads(index.data.decode('utf-8'))
        for endpoint in endpoints:
            collection_name = "/".join(endpoints[endpoint].split(
                "/{}/".format(self.API_NAME))[1:])
            if collection_name in self.doc.collections:
                response_get = self.client.get(endpoints[endpoint])
                assert response_get.status_code == 200
                response_get_data = json.loads(
                    response_get.data.decode('utf-8'))
                assert "search" in response_get_data
                assert "mapping" in response_get_data["search"]
                collection = self.doc.collections[collection_name]["collection"]
                class_ = self.doc.parsed_classes[collection.class_.title]["class"]
                class_props = [x.prop for x in class_.supportedProperty]
                for mapping in response_get_data["search"]["mapping"]:
                    if mapping["property"] not in ["limit", "offset", "pageIndex"]:
                        assert mapping["property"] in class_props 

# 需要导入模块: import json [as 别名]
# 或者: from json import loads [as 别名]
def test_config_to_json_string(self):
        config = modeling.BertConfig(vocab_size=99, hidden_size=37)
        obj = json.loads(config.to_json_string())
        self.assertEqual(obj["vocab_size"], 99)
        self.assertEqual(obj["hidden_size"], 37)