Python demjson.decode方法代码示例

# 需要导入模块: import demjson [as 别名]
# 或者: from demjson import decode [as 别名]
def jsonp2dict(jsonp):
    """
    解析jsonp类型的返回
    :param jsonp:
    :return:
    """

    try:
        l_index = jsonp.index("(") + 2
        r_index = jsonp.rindex(")") - 1
        jsonp_info = jsonp[l_index:r_index]
    except ValueError:
        logger.error("Input is not in a jsonp format. %s" % jsonp)
        return
    try:
        return demjson.decode(jsonp_info)
    except demjson.JSONDecodeError as e:
        if jsonp_info == "new Boolean(true)":
            return True
        elif jsonp_info == "null":
            return None
        else:
            logger.error("解析jsonp返回失败")
            logger.error(jsonp)
            raise e 

# 需要导入模块: import demjson [as 别名]
# 或者: from demjson import decode [as 别名]
def run(self):
        # 抓取沪市 ETF 列表
        url = 'http://query.sse.com.cn/commonQuery.do?sqlId=COMMON_SSE_ZQPZ_ETFLB_L_NEW'
        response = requests.get(url, headers=DEFAULT_SH_ETF_LIST_HEADER)
        response_dict = demjson.decode(response.text)

        df = pd.DataFrame(response_dict.get('result', []))
        self.persist_etf_list(df, exchange='sh')
        self.logger.info('沪市 ETF 列表抓取完成...')

        # 抓取沪市 ETF 成分股
        self.download_sh_etf_component(df)
        self.logger.info('沪市 ETF 成分股抓取完成...')

        # 抓取深市 ETF 列表
        url = 'http://www.szse.cn/api/report/ShowReport?SHOWTYPE=xlsx&CATALOGID=1945'
        response = requests.get(url)

        df = pd.read_excel(io.BytesIO(response.content), dtype=str)
        self.persist_etf_list(df, exchange='sz')
        self.logger.info('深市 ETF 列表抓取完成...')

        # 抓取深市 ETF 成分股
        self.download_sz_etf_component(df)
        self.logger.info('深市 ETF 成分股抓取完成...') 

# 需要导入模块: import demjson [as 别名]
# 或者: from demjson import decode [as 别名]
def _get_fund_list_onepage(self, company='', page_no = 1, page_size = 100):
        url    = 'http://fund.eastmoney.com/Data/Fund_JJJZ_Data.aspx?page=%d,%d&gsid=%s' % (page_no, page_size, company)
        prefix = 'var db='
        response = self.do_request(url)
        response = self._get_and_parse_js(url, prefix)
        if response is None:
            return None, '获取数据失败'

        jsonobj = demjson.decode(response)
        rsp = jsonobj['datas']
        datestr = jsonobj['showday']
        df = pd.DataFrame(rsp)
        if len(df) > 0:
            df.drop(df.columns[5:], axis=1, inplace=True)
            df.columns = ['fundcode', 'fundname', 'pingyin', 'nav', 'accu_nav']
            df['date'] = datestr[0]
            return df, ''
        else:
            return None, '' 

# 需要导入模块: import demjson [as 别名]
# 或者: from demjson import decode [as 别名]
def _get_fundlist_by_type_page(self, type, page_no = 1, page_size = 100):
        url    = 'http://fund.eastmoney.com/Data/Fund_JJJZ_Data.aspx?page=%d,%d' % (page_no, page_size)
        prefix = 'var db='
        type_param = fund_type[type]

        response = self._get_and_parse_js(url, prefix, param=type_param)
        jsonobj = demjson.decode(response)
        rsp = jsonobj['datas']
        datestr = jsonobj['showday']
        df = pd.DataFrame(rsp)
        if len(df) > 0:
            df.drop(df.columns[5:], axis=1, inplace=True)
            df.columns = ['fundcode', 'fundname', 'pingyin', 'nav', 'accu_nav']
            df['date'] = datestr[0]
            return df, ''
        else:
            return None, '获取数据失败' 

# 需要导入模块: import demjson [as 别名]
# 或者: from demjson import decode [as 别名]
def download_sina_category_detail(self, response):
        if response.text == 'null' or response.text is None:
            return
        category_jsons = demjson.decode(response.text)
        for category in category_jsons:
            self.file_lock.acquire()
            if get_exchange(category['code']) == 'sh':
                df = self.sh_df
            elif get_exchange(category['code']) == 'sz':
                df = self.sz_df

            if category['code'] in df.index:
                current_ind = df.at[category['code'], self.category_type]

                if type(current_ind) == list and (response.meta['ind_name'] not in current_ind):
                    current_ind.append(response.meta['ind_name'])

                elif type(current_ind) == str and response.meta['ind_name'] != current_ind:
                    current_ind = [current_ind, response.meta['ind_name']]
                else:
                    current_ind = response.meta['ind_name']

                df.at[category['code'], self.category_type] = current_ind
            self.file_lock.release() 

# 需要导入模块: import demjson [as 别名]
# 或者: from demjson import decode [as 别名]
def makeRequest(symbol):
  url = 'http://www.google.com/finance/company_news?output=json&q=' \
        + symbol + '&start=0&num=1000'
  try:
    req = Request(url)
    resp = urlopen(req)
    content = resp.read()

    content_json = demjson.decode(content)
    clusters = content_json['clusters']
    if clusters[0]:
      return clusters[0]["a"]
    return []
  except urllib.error.URLError as e:
    return []
  except urllib.error.HTTPError as e:
    return [] 

# 需要导入模块: import demjson [as 别名]
# 或者: from demjson import decode [as 别名]
def stock_sector_detail(sector: str = "hangye_ZL01") -> pd.DataFrame:
    """
    新浪行业-板块行情-成份详情
    http://finance.sina.com.cn/stock/sl/#area_1
    :param sector: stock_sector_spot 返回的 label 值, choice of {"新浪行业", "概念", "地域", "行业"}; "启明星行业" 无详情
    :type sector: str
    :return: 指定 sector 的板块详情
    :rtype: pandas.DataFrame
    """
    url = "http://vip.stock.finance.sina.com.cn/quotes_service/api/json_v2.php/Market_Center.getHQNodeData"
    params = {
        "page": "1",
        "num": "40",
        "sort": "symbol",
        "asc": "1",
        "node": sector,
        "symbol": "",
        "_s_r_a": "init",
    }
    r = requests.get(url, params=params)
    temp_df = pd.DataFrame(demjson.decode(r.text))
    return temp_df 

# 需要导入模块: import demjson [as 别名]
# 或者: from demjson import decode [as 别名]
def stock_zh_ah_spot() -> pd.DataFrame:
    """
    腾讯财经-港股-AH-实时行情
    https://stockapp.finance.qq.com/mstats/#mod=list&id=hk_ah&module=HK&type=AH&sort=3&page=3&max=20
    :return: 腾讯财经-港股-AH-实时行情
    :rtype: pandas.DataFrame
    """
    big_df = pd.DataFrame()
    page_count = _get_zh_stock_ah_page_count() + 1
    for i in tqdm(range(1, page_count)):
        hk_payload.update({"reqPage": i})
        res = requests.get(hk_url, params=hk_payload, headers=hk_headers)
        data_json = demjson.decode(res.text[res.text.find("{"): res.text.rfind("}") + 1])
        big_df = big_df.append(pd.DataFrame(data_json["data"]["page_data"]).iloc[:, 0].str.split("~", expand=True), ignore_index=True).iloc[:, :-1]
    big_df.columns = ["代码", "名称", "最新价", "涨跌幅", "涨跌额", "买入", "卖出", "成交量", "成交额", "今开", "昨收", "最高", "最低"]
    return big_df 

# 需要导入模块: import demjson [as 别名]
# 或者: from demjson import decode [as 别名]
def stock_zh_ah_name() -> dict:
    """
    腾讯财经-港股-AH-股票名称
    :return: 股票代码和股票名称的字典
    :rtype: dict
    """
    big_df = pd.DataFrame()
    page_count = _get_zh_stock_ah_page_count() + 1
    for i in tqdm(range(1, page_count)):
        hk_payload.update({"reqPage": i})
        res = requests.get(hk_url, params=hk_payload, headers=hk_headers)
        data_json = demjson.decode(res.text[res.text.find("{"): res.text.rfind("}") + 1])
        big_df = big_df.append(pd.DataFrame(data_json["data"]["page_data"]).iloc[:, 0].str.split("~", expand=True), ignore_index=True).iloc[:, :-1]
    big_df.columns = ["代码", "名称", "最新价", "涨跌幅", "涨跌额", "买入", "卖出", "成交量", "成交额", "今开", "昨收", "最高", "最低"]
    code_name_dict = dict(zip(big_df["代码"], big_df["名称"]))
    return code_name_dict 

# 需要导入模块: import demjson [as 别名]
# 或者: from demjson import decode [as 别名]
def stock_zh_kcb_spot() -> pd.DataFrame:
    """
    从新浪财经-A股获取所有A股的实时行情数据, 大量抓取容易封IP
    http://vip.stock.finance.sina.com.cn/mkt/#qbgg_hk
    :return: pandas.DataFrame
    """
    big_df = pd.DataFrame()
    page_count = get_zh_kcb_page_count()
    zh_sina_stock_payload_copy = zh_sina_kcb_stock_payload.copy()
    for page in tqdm(range(1, page_count+1)):
        zh_sina_stock_payload_copy.update({"page": page})
        res = requests.get(
            zh_sina_kcb_stock_url,
            params=zh_sina_kcb_stock_payload)
        data_json = demjson.decode(res.text)
        big_df = big_df.append(pd.DataFrame(data_json), ignore_index=True)
    return big_df 

# 需要导入模块: import demjson [as 别名]
# 或者: from demjson import decode [as 别名]
def bond_zh_hs_cov_spot() -> pd.DataFrame:
    """
    新浪财经-债券-沪深可转债的实时行情数据, 大量抓取容易封IP
    http://vip.stock.finance.sina.com.cn/mkt/#hskzz_z
    :return: 所有沪深可转债在当前时刻的实时行情数据
    :rtype: pandas.DataFrame
    """
    big_df = pd.DataFrame()
    page_count = _get_zh_bond_hs_cov_page_count()
    zh_sina_bond_hs_payload_copy = zh_sina_bond_hs_cov_payload.copy()
    for page in tqdm(range(1, page_count + 1)):
        zh_sina_bond_hs_payload_copy.update({"page": page})
        res = requests.get(zh_sina_bond_hs_cov_url, params=zh_sina_bond_hs_payload_copy)
        data_json = demjson.decode(res.text)
        big_df = big_df.append(pd.DataFrame(data_json), ignore_index=True)
    return big_df 

# 需要导入模块: import demjson [as 别名]
# 或者: from demjson import decode [as 别名]
def bond_zh_hs_spot():
    """
    新浪财经-债券-沪深债券的实时行情数据, 大量抓取容易封IP
    http://vip.stock.finance.sina.com.cn/mkt/#hs_z
    :return: 所有沪深债券在当前时刻的实时行情数据
    :rtype: pandas.DataFrame
    """
    big_df = pd.DataFrame()
    page_count = get_zh_bond_hs_page_count()
    zh_sina_bond_hs_payload_copy = zh_sina_bond_hs_payload.copy()
    for page in range(1, page_count + 1):
        print(page)
        zh_sina_bond_hs_payload_copy.update({"page": page})
        res = requests.get(zh_sina_bond_hs_url, params=zh_sina_bond_hs_payload_copy)
        data_json = demjson.decode(res.text)
        big_df = big_df.append(pd.DataFrame(data_json), ignore_index=True)
    return big_df 

# 需要导入模块: import demjson [as 别名]
# 或者: from demjson import decode [as 别名]
def match_main_contract(exchange="dce"):
    subscribe_cffex_list = []
    exchange_symbol_list = zh_subscribe_exchange_symbol(exchange).iloc[:, 1].tolist()
    for item in exchange_symbol_list:
        zh_match_main_contract_payload.update({"node": item})
        res = requests.get(
            zh_match_main_contract_url, params=zh_match_main_contract_payload
        )
        data_json = demjson.decode(res.text)
        data_df = pd.DataFrame(data_json)
        try:
            main_contract = data_df.iloc[0, :3]
            subscribe_cffex_list.append(main_contract)
        except:
            print(item, "无主力连续合约")
            continue
    print("主力连续合约获取成功")
    return pd.DataFrame(subscribe_cffex_list) 

# 需要导入模块: import demjson [as 别名]
# 或者: from demjson import decode [as 别名]
def zh_subscribe_exchange_symbol(exchange="dce"):
    res = requests.get(zh_subscribe_exchange_symbol_url)
    data_json = demjson.decode(
        res.text[res.text.find("{"): res.text.find("};") + 1])
    if exchange == "czce":
        data_json["czce"].remove("郑州商品交易所")
        return pd.DataFrame(data_json["czce"])
    if exchange == "dce":
        data_json["dce"].remove("大连商品交易所")
        return pd.DataFrame(data_json["dce"])
    if exchange == "shfe":
        data_json["shfe"].remove("上海期货交易所")
        return pd.DataFrame(data_json["shfe"])
    if exchange == "cffex":
        data_json["cffex"].remove("中国金融期货交易所")
        return pd.DataFrame(data_json["cffex"]) 

# 需要导入模块: import demjson [as 别名]
# 或者: from demjson import decode [as 别名]
def match_main_contract(exchange="dce"):
    subscribe_cffex_list = []
    exchange_symbol_list = zh_subscribe_exchange_symbol(
        exchange).iloc[:, 1].tolist()
    for item in exchange_symbol_list:
        zh_match_main_contract_payload.update({"node": item})
        res = requests.get(
            zh_match_main_contract_url,
            params=zh_match_main_contract_payload)
        data_json = demjson.decode(res.text)
        data_df = pd.DataFrame(data_json)
        try:
            main_contract = data_df[data_df.iloc[:, 3:].duplicated()]
            print(main_contract["symbol"].values[0])
            subscribe_cffex_list.append(main_contract["symbol"].values[0])
        except:
            print(item, "无主力合约")
            continue
    print("主力合约获取成功")
    return ','.join(["nf_" + item for item in subscribe_cffex_list])