Python data.DataReader类代码示例

def pull_stocks_data(retries=2, start_date=None, end_date=None):
    """
    Pulling stocks raw data, of the stocks in the symbol list.
    :param retries: number of retries for getting each stock's data
    :param start_date: the first day of the data (datetime format), default value is 2 years before end_date.
    :param end_date: the last day of the data (datetime format), default value is today
    """
    symbols = get_stocks_symbols(write_to_files=False)
    log.notice("Starting to pull stocks data")
    end_date = datetime.today() if end_date is None else end_date
    start_date = end_date - timedelta(365*2)  # take as a default 2 years backwards

    for retry in range(retries):
        for symbol in symbols:
            filepath = make_filepath(DATA_PATH+"symbols", symbol, 'csv')  # optimize by avoiding calling this function every time
            try:
                data = DataReader(symbol,  'yahoo', start_date, end_date, retry_count=1)
            except IOError as e:
                log.error("IOError for data query of symbol: {}\n\tError msg: {}".format(symbol, e))
                continue
            data.to_csv(filepath)
            symbols.pop(symbols.index(symbol))
        log.warning("Unable to get {} symbols on try #{}".format(len(symbols), retry+1))

    log.error("Unable to get {} symbols after {} retries:\n{}".format(len(symbols), retries, symbols))

def main():
    #    db = psql.get_db()
    #    product_codes = db.prepare('''SELECT p.code AS code, p.id as id
    #        FROM products p LEFT JOIN companies c ON p.company_id = c.id
    #        WHERE c.sector IS NOT NULL
    #        and p.id < 9354
    #        ORDER BY p.id''')()

    product_codes = get_components_yahoo('^DJI').index
    total = len(product_codes)
    cur = 0
    products=[]

    logger.info('Start downloading of %d products.', len(product_codes))
    for code in product_codes:
        cur += 1
        try:
            product = DataReader(code, 'yahoo', start=date_from, end=date_to)
        except:
            logger.warn('(%d/%d) Fail downloading %s.', cur, total, code)
            continue

        product.code = code
        products.append(product)
        logger.info('(%d/%d) Downloaded %s.', cur, total, code)

    logger.info('Download complete.')
    return opt(products)

def refreshYahooDatabase(db):
    
    rs = yahoo.yahooRecordset(db)
    ts = timeseries.timeseriesRecordset(db)
    
    # step 1: get the yahoo index end date
    idx = rs.select('*')
    
    for i in idx:
        
        try:
            r = ts.getLastDate(i['id'])
            
            start = r[0]['max_date']
            
            data = pd.DataFrame()
            
            if start != None:
                start = start + dt.timedelta(days=1)
                
                data = DataReader(  str(i['key']), 'yahoo', start)
            else:
                data = DataReader(  str(i['key']), 'yahoo')
            
            # insert new data
            add = []
            
            for index, row in data.iterrows():
                add.append((i['id'], index, row[i['field']]))
                
            ts.insert(add)
            
        except Exception as e:
            print('error processing index ' + str(i['key'] \
                + '/' + str(i['field']) + '\n'))

def save_data():
    start = '1/1/1990'

    # Get S&P 500 data from yahoo
    sp500 = get_data_yahoo('^GSPC', start=start)['Adj Close']
    sp500.name = 'SP500'

    vix = get_data_yahoo('^VIX', start=start)['Adj Close']
    vix.name = 'VIX'

    # Get ten year and 3 month t-bill rates
    ten_yr = DataReader('DGS10', 'fred', start=start)
    three_mon = DataReader('DGS3MO', 'fred', start=start)

    ten_yr = ten_yr.ix[ten_yr.DGS10.str.count(r'^\.') != 1].astype(float)
    three_mon = three_mon.ix[three_mon.DGS3MO.str.count(r'^\.') != 1].astype(float)

    data = ten_yr.join(three_mon)
    data = data.join(sp500)
    data = data.join(vix)

    # Drop non-like observations (obs on different days)
    data = data.dropna()

    data.save('SP_YC.db')
    data.to_csv('the_data.csv')

 def readData(self, lookupTicker, source, start, end):
     
     '''Read the data - assumes start and end are datetime.date objects'''
     
     try:  
         lookupTicker = str(lookupTicker)
         if source == 'Quandl':
             #use Quandl reader
             start = str(start)
             end = str(end)
             data = Quandl.get(lookupTicker,
                               authtoken = self.quandlAuthToken,
                               trim_start = start, 
                               trim_end= end)
         else:
             #use pandas.io DataReader
             data = DataReader(lookupTicker, source , start, end)
             
         data = data.reset_index()
         logging.info("Read ticker {}".format(lookupTicker))
     except:
         logging.error("importData: Can't read ticker {}".format(lookupTicker))
         raise
     else:
         return data

def getTimewindowStockPrice(ticker):
    timewindowStockPrice = {}#key: year; value: weekly price
    stockDF = DataReader(ticker,  "google", "2009-01-01", datetime.today().date())
#     print stockDF
    for idx, row in stockDF.iterrows():
#         print row[0], row['Close']
#         print datetime.fromtimestamp(idx)
#         print str(idx)
        dt = dateutil.parser.parse(str(idx)).date()
        year = dt.isocalendar()[0]
        week = dt.isocalendar()[1]
        price = row['Close']
        print year, week, price
        if year not in timewindowStockPrice:
            timewindowStockPrice[year] = {}
        if week not in timewindowStockPrice[year]:
            timewindowStockPrice[year][week] = []
#         print row['Close']
        timewindowStockPrice[year][week].append(price)
        
    #normalized weekly price
    for year in timewindowStockPrice.keys():
        for week in timewindowStockPrice[year].keys():
            timewindowStockPrice[year][week] = scipy.mean(timewindowStockPrice[year][week])
    
#     for year in timewindowStockPrice.keys():
#         print timewindowStockPrice[year]
    return timewindowStockPrice

 def get_riskfree_rate(self,startdate,enddate,freq="M",maturity='1M'):
     """
     Rates from FRED
     http://research.stlouisfed.org/fred2/categories/116
     """
     rfcache = self.__class__._cacherfrate
     grabdata = False
     if rfcache == None:
         grabdata = True
     elif rfcache[0]< startdate or rfcache[1] > enddate:
         grabdata = True
          
     if grabdata:
         dt          = DataReader('DTB4WK',"fred", startdate,enddate)
         dt.columns  = ['RFRate']
         dt.fillna(method='backfill',inplace=True)
         rfcache     = (startdate,enddate,dt)
         self.__class__._cacherfrate= rfcache
     else:
         dt          = rfcache[2]
     
     dsm     = dt[startdate:enddate].resample('M')
     return dsm
     
     
     
     
     

def get_data_from_yahoo(ticker):
    # No older computations OK
    start_date = date(2010, 01, 01)
    end_date = date.today()
    df = DataReader(ticker, 'yahoo', start_date, end_date)
    df.to_pickle(TMP_dir + ticker)
    print "Getting data for " + ticker + " from yahoo"
    return df

def getPayOff(year,month):
    nifty = DataReader("^NSEI","yahoo",datetime(year,month,1),datetime(year,month,25))
    bn=DataReader("^NSEBANK","yahoo",datetime(year,month,1),datetime(year,month,25))

    nifty_daily=nifty.diff()['Close']
    bn_daily=bn.diff()['Close']

    nifty_daily_return =50*(nifty_daily)
    bn_daily_return =25*(bn_daily)
    return -nifty_daily_return.sum()+bn_daily_return.sum()

def hist_vol(sym, days=10):
    try:
        quotes = DataReader(sym, 'yahoo')['Close'][-days:]
    except Exception:
        print "Problem getting historical volatility!"
        raise SystemExit(code)
        return None, None
    logreturns = np.log(quotes / quotes.shift(1))
    vol = np.sqrt(252*logreturns.var()) #252 trading days in year (annualized volatility)
    return float(vol)

def getLongShortPayOff(year,month,longSymbol,longQty,shortSymbol,shortQty):
    shortStk = DataReader(shortSymbol,"yahoo",datetime(year,month,1),datetime(year,month,25))
    longStk=DataReader(longSymbol,"yahoo",datetime(year,month,1),datetime(year,month,25))

    short_daily=shortStk.diff()['Close']
    long_daily=longStk.diff()['Close']

    short_daily_return =shortQty*(short_daily)
    long_daily_return =longQty*(long_daily)
    #print abs(long_daily_return.sum())-abs(short_daily_return.sum())
    return abs(long_daily_return.sum())-abs(short_daily_return.sum())

    def ts(self, symbol):
        parse = self.parsesymbol(symbol)
        df    = DataReader(parse['eq'], parse['proto'],start=datetime.datetime(1950,1,1))
        df    = df.rename(columns=lambda x: '_'.join(x.split()).lower()) # Need for Adj Close :(
        #print df.columns
        ts = df[parse['hlocv']]
        ts.index = map(lambda x: x.date(), ts.index)
        ts.name  = parse['eq']
        if '@' in symbol:
	    ts.name += '@%s' % (parse['hlocv'])
        return ts

def main(symbol):

    t1 = datetime.now()
    t2 = t1 - timedelta(days=PAST_DAYS)

    df  = DataReader(symbol,  FIN_SERVICE_PROVIDER , t2, t1)
    print df.head()
    print '...' * 20
    print df.tail()

    return df

def _load_raw_yahoo_data(indexes=None, stocks=None, start=None, end=None):
    """Load closing prices from yahoo finance.

    :Optional:
        indexes : dict (Default: {'SPX': '^GSPC'})
            Financial indexes to load.
        stocks : list (Default: ['AAPL', 'GE', 'IBM', 'MSFT',
                                 'XOM', 'AA', 'JNJ', 'PEP', 'KO'])
            Stock closing prices to load.
        start : datetime (Default: datetime(1993, 1, 1, 0, 0, 0, 0, pytz.utc))
            Retrieve prices from start date on.
        end : datetime (Default: datetime(2002, 1, 1, 0, 0, 0, 0, pytz.utc))
            Retrieve prices until end date.

    :Note:
        This is based on code presented in a talk by Wes McKinney:
        http://wesmckinney.com/files/20111017/notebook_output.pdf
    """

    assert indexes is not None or stocks is not None, """
must specify stocks or indexes"""

    if start is None:
        start = pd.datetime(1990, 1, 1, 0, 0, 0, 0, pytz.utc)

    if start is not None and end is not None:
        assert start < end, "start date is later than end date."

    data = OrderedDict()

    if stocks is not None:
        for stock in stocks:
            print(stock)
            stock_pathsafe = stock.replace(os.path.sep, '--')
            cache_filename = "{stock}-{start}-{end}.csv".format(
                stock=stock_pathsafe,
                start=start,
                end=end).replace(':', '-')
            cache_filepath = get_cache_filepath(cache_filename)
            if os.path.exists(cache_filepath):
                stkd = pd.DataFrame.from_csv(cache_filepath)
            else:
                stkd = DataReader(stock, 'yahoo', start, end).sort_index()
                stkd.to_csv(cache_filepath)
            data[stock] = stkd

    if indexes is not None:
        for name, ticker in iteritems(indexes):
            print(name)
            stkd = DataReader(ticker, 'yahoo', start, end).sort_index()
            data[name] = stkd

    return data

 def importData():
     
     #Start Time
     start = datetime(2010,1,1)
     end = datetime.date(datetime.now())
     data = DataReader(sp500constituents[0], "yahoo", start, end)
     
     
     en = enumerate(sp500constituents)
     [i for i, x in en if x=='WFMI']
     
     
     sp500constituents[200:len(sp500constituents)]
     problems = []
     dataImportProblems = []
     for series in sp500constituents[485:len(sp500constituents)]:
         print series 
         try:  
             data = DataReader(series, "yahoo", start, end)
             data = data.reset_index()
         except:
             print "Can't read {}".format(series)
             dataImportProblems.append(series)
             continue
         con = sqlite3.connect("/home/phcostello/Documents/Data/FinanceData.sqlite")
         try:
             psql.write_frame( data, series, con)
             con.commit()
         except:
             print "Problems with {}".format(series)
             problems.append(series)
         finally:
             con.close()
     
     #changing tables to have date formats so RODBC driver recognizes
     #Should check that this is occuring above.
     con = sqlite3.connect("/home/phcostello/Documents/Data/FinanceData.sqlite")
     for tb in sp500constituents:
         if psql.has_table(tb, con):
             sqltxt = "SELECT * FROM {}".format(tb)
             #print sqltxt
             data = psql.read_frame(sqltxt, con)
             sqlDropTxt = 'DROP TABLE "main"."{}"'.format(tb)
             #print sqlDropTxt
             psql.execute(sqlDropTxt, con)
             con.commit()
             psql.write_frame( data, tb, con)
             con.commit()
     
     con.close()