Python Request.decode方法代码示例

# 需要导入模块: from urllib.request import Request [as 别名]
# 或者: from urllib.request.Request import decode [as 别名]
 def extract_bitinfocharts(self, coin, metric="price", coin_col=False, metric_col=False):
     """Retrieve historical data for a specific cyrptocurrency scraped from bitinfocharts.com
     
     Parameters
     ----------
     coin : the code of the cryptocurrency (e.g. 'btc' for bitcoin)
         full range of available coins can be found on bitinfocharts.com
     metric : the particular coin information to be retrieved
         (options are limited to those listed on bitinfocharts.com
         including 'price', 'marketcap', 'transactions' and 'sentinusd'
     coin_col : whether to include the coin name as a column
         (default is False i.e. the column is not included)
     metric_col : whether to include the metric name as a column
         (default is False i.e. the column is not included)
         
     Returns
     -------
     pandas Dataframe
     """
     if coin not in ['btc', 'eth', 'xrp', 'bch', 'ltc', 'dash', 'xmr', 'btg', 'etc', 'zec', 
                     'doge', 'rdd', 'vtc', 'ppc', 'ftc', 'nmc', 'blk', 'aur', 'nvc', 'qrk', 'nec']:
         raise ValueError("Not a valid coin")
     if metric not in ['transactions', 'size', 'sentbyaddress', 'difficulty', 'hashrate', 'price', 
                       'mining_profitability', 'sentinusd', 'transactionfees', 'median_transaction_fee', 
                     'confirmationtime', 'marketcap', 'transactionvalue', 'mediantransactionvalue',
                      'tweets', 'activeaddresses', 'top100cap']:
         raise ValueError("Not a valid bitinfocharts metric")
     new_col_name = "_".join([coin, metric])
     parsed_page = Request("https://bitinfocharts.com/comparison/{}-{}.html".format(metric, coin),
                         headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11'})
     try: 
         parsed_page = urlopen(parsed_page, timeout=self.timeout).read()
         parsed_page = parsed_page.decode("utf8")
     except Exception as e:
         return pd.DataFrame({"error":e}, index=[0])
     start_segment = parsed_page.find("new Dygraph")
     if start_segment != -1:
         start_list = parsed_page.find('[[', start_segment)
         end_list = parsed_page.find(']]', start_list)
         parsed_page = parsed_page[start_list:end_list]
     else:
         return pd.DataFrame({"error":"Could not find the appropriate text tag"}, index=[0])
     parsed_page = parsed_page.replace('new Date(', '')
     parsed_page = parsed_page.replace(')', '')
     parsed_page = parsed_page.replace('null', '0')
     parsed_page = parsed_page.replace('["', '{"date":"')
     parsed_page = parsed_page.replace('",', '","{}":'.format(new_col_name))
     parsed_page = parsed_page.replace('],', '},')
     parsed_page = parsed_page + '}]'
     output = json.loads(parsed_page)
     output = pd.DataFrame(output)
     output['date'] = pd.to_datetime(output['date'], format="%Y-%m-%d")
     output = output[(output['date']>=self.from_date) & (output['date']<=self.to_date)]
     # for consistency, put date column first
     output = output[['date', new_col_name]]
     if coin_col:
         output['coin'] = coin
     if metric_col:
         output['metric'] = metric
     return output.sort_values(by='date', ascending=self.ascending).reset_index(drop=True)

# 需要导入模块: from urllib.request import Request [as 别名]
# 或者: from urllib.request.Request import decode [as 别名]
def main():
    try:
        sys.argv[5]
    except IndexError:
        srv = 2181
    else:
        srv = sys.argv[5]

    if sys.argv[1] == "normal":
        pingms = sys.argv[4]
        dlkbps = sys.argv[2]
        ulkbps = sys.argv[3]
    elif sys.argv[1] == "smart":
        dlkbps = floor(
            float(sys.argv[2]) * float(randrange(9450000, 9750000)) / 10000000)
        ulkbps = floor(
            float(sys.argv[3]) * float(randrange(9450000, 9750000)) / 10000000)
        ulkbps = floor(ulkbps * float(randrange(9350, 9650)) / 10000)
        if int(sys.argv[4]) >= 4:
            pingms = int(sys.argv[4]) + randrange(-3, 10)
        else:
            pingms = sys.argv[4]
    else:
        print("This is madness!")
        sys.exit()

    rawRequest = [
        'download=%s' % int(dlkbps),
        'ping=%s' % pingms,
        'upload=%s' % int(ulkbps),
        'promo=',
        'startmode=%s' % 'pingselect',
        'recommendedserverid=%s' % srv,
        'accuracy=%s' % 1,
        'serverid=%s' % srv,
        'hash=%s' % md5(('%s-%s-%s-%s' %
                         (pingms, int(ulkbps), int(dlkbps), '297aae72'))
                        .encode()).hexdigest()]

    req = Request('http://www.speedtest.net/api/api.php',
                  data='&'.join(rawRequest).encode())
    req.add_header('Referer', 'http://c.speedtest.net/flash/speedtest.swf')
    x = urlopen(req)
    req = x.read()
    code = x.code
    x.close()

    reqresult = parse_qs(req.decode())
    results = reqresult.get('resultid')

    print('Image: http://www.speedtest.net/result/%s.png' %
          results[0])
    print('In a webpage: http://www.speedtest.net/my-result/%s' % results[0])

# 需要导入模块: from urllib.request import Request [as 别名]
# 或者: from urllib.request.Request import decode [as 别名]
for i in ['旅游','出境游','国内游','周边游','旅行']:
    try:
        with open(i+'360.txt') as f:
            with open(i+'.txt', 'a') as g:
                for j in f:
                    j = j.strip()
                    if j in app:
                        print({'keyword':i, 'name':j, 'detail':app[j]}, file=g)
                    else:
                        try:
                            print(j)
                            data = {'kw': j}
                            data = baseUrl + urlencode(data)
                            data = Request(url=data, headers=genHeader())
                            data = urlopen(data).read()
                            sleep(3)
                            data = data.decode()
                            data = loads(data)
                            for k in data['obj']['items']:
                                if j in k['appDetail']['appName'] or k['appDetail']['appName'] in j:
                                    print(k['appDetail']['appName'])
                                    print({'keyword':i, 'name':j, 'detail':k['appDetail']}, file=g)
                                    app[j] = k['appDetail']
                                    break
                        except Exception as e:
                            print('inner', str(e))
                            continue
    except Exception as e:
        print('outer', str(e))
        continue