Python spider.log_with_time函数代码示例

def stock1_parser(task, rule):
    try:
        j = demjson.decode(task['text'])
    except:
        log_with_time("bad response: %r"%task['url'])
        return []
    code = j['code']
    message = j['message']

    url = ""
    ret = {"spider":[], 'stock2':[]}

    if code == 3 and message:
        try:
            skuid = re.search("\d+", message).group()
            url = surl2(task['gid'], skuid)
        except:
            return []
    if url == "":
        #print(task['text'])
        stock = 1 if j.get('totalAmount') else 0
        ret['spider'] = format_price([(itemurl+task['gid'], task['price'], stock)])
    else:
        ret['stock2'] = [(url, task['gid'], task['price'])]

    return ret

def list_parser(task, rule):
    t = etree.HTML(task["text"]) 
    nodes = t.xpath(rule["node"]) 
    if not nodes:
        log_with_time("node rule error: %s" % task["url"])
        return 
    dp = []
    dps = {}
    ret = [] 
    now = int(time.time()) 
    for node in nodes:
        link = node.xpath(rule["link"])
        gid = node.xpath(rule["gid"]) 
        if not link or not gid:
            log_with_time("rule error: %s" % task["url"])
            continue 
        gid = gid[0]
        dp.append((link[0], ""))
        ret.append(gid)
        dps[gid] = now 
    return {
            "dps_log": dps,
            "dp": dp,
            "price": ret,
            } 

def cats_parser(url, res,  rule):
    content = res['text']
    t = etree.HTML(content)
    ret = set()
    items = t.xpath(rule)
    for v in items:
        #pdb.set_trace()
        if '/c0-0/' in v:
            continue
        if '/ctg/s2/' in v:
            r = "(?<=/ctg/s2/).+"
            cat = re.search(r, v)
            if not cat:
                log_with_time("bad regex: %r %r" % (r, v))
                continue
            cat = cat.group().split('-')[0]
            ret.add(ctgurl % cat)
        elif 'list.yhd.com' in v:
            # http://list.yhd.com/.../
            r = "(?<=yhd\.com\/).+"
            cat = re.search(r, v)
            if not cat:
                log_with_time("bad regex: %r %r" % (r, v))
                continue
            cat = cat.group().split('-')[0]
            ret.add(lsturl % cat)
    return ret

def rt_parser(items): 
    pids = get_pids(items)
    if not pids:
        log_with_time("got nothing: %s" % entries)
        return
    purl = price_url % (",".join(["J_" + i for i in pids]), 
            random.randint(1000000, 10000000), int(time.time() * 1000)) 
    surl = stock_url % (async_http.quote(",".join([i for i in pids])), 
            random.randint(1000000, 10000000), int(time.time() * 1000)) 

    price_res = simple_http.get(purl) 
    stock_res = simple_http.get(surl)
    if price_res["status"] != 200 or stock_res["status"] != 200:
        log_with_time("not200: %s" % price["res"])
        return
    try:
        price_json = jsonp_json(price_res["text"]) 
        stock_json = jsonp_json(stock_res["text"].decode("gbk"))
    except: 
        traceback.print_exc()
        return
    prices = {} 
    for i in price_json: 
        prices[i["id"].split("_")[1]] = i["p"]
    stocks = {} 
    for k,v in stock_json.items(): 
        s = v["StockStateName"]
        if u"有货" in s or u"现货" in s:
            stocks[k] = 1
        else:
            stocks[k] = 0 
    ret = []
    for pid in prices:
        ret.append((str(pid), str(prices[pid]), stocks[pid])) 
    return format_price(ret)

def dp_parser(task, rule): 
    desc_url = re.findall("desc: '(http.*?desc/[0-9]+)'", task["text"]) 
    if not desc_url:
        log_with_time("no desc: %s" % task["url"])
        return
    crc = urlcrc.get_urlcrc(3, task["url"])
    return [(desc_url[0], str(crc), "")] 

def list_parser(task, rule):
    t = etree.HTML(task['text'])
    nodes = t.xpath(rule['nodes'])
    prices = []
    items = []
    dps = {}
    #pdb.set_trace()
    for node in nodes:
        gid = node.attrib['itemid']
        buyinfo = node.xpath(rule['buyinfo'])
        if not gid:
            log_with_time("bad response: %r"%task['url'])
            continue
        if buyinfo:
            buyinfo = buyinfo[0]
            buycart = buyinfo.xpath(rule['buycart'])
            stock = 1
            if not buycart:
                if buyinfo.xpath(rule['sellout']) or not node.xpath(rule['comment']):
                    stock = 0
            prices.append((gid, stock))
        else:
            items.append(gid)
        dps[gid] = int(time.time())
    return {"prices": prices, "items": items, "dps": dps}

def extract_book(url, tree, rule): 
    result = []
    dps = [] 
    now = int(time.time())
    dps_log = {} 
    nodes = tree.xpath(rule["book_node"])
    comments = {}
    lid = re.search("\d+", url.split('-')[-1]).group()
    for node in nodes:
        link_node = node.xpath(rule["book_title"]) 
        stock = node.xpath(rule["book_stock"]) 
        comment = node.xpath(rule["book_comment"])
        if not link_node or not stock: 
            log_with_time("rule error: %s" % url)
            continue 
        link_node = link_node[0]
        link = link_node.attrib["href"]
        gid = re_gid.search(link).group()
        comments[gid] = comment[0]
        title = link_node.text
        if u"有货" in stock[0]:
            s = 1
        else:
            s = 0 
        dps_log[gid] = now
        dps.append((link, gid, title))
        result.append((link, gid, lid, s)) 
    return {
            "book_price": result,
            #"dp": dps,
            "dps_log": dps_log,
            "comment": comments
            } 

def fix_url(url):
    if "tuan" in url:
        log_with_time("skip url: %s" % url)
        return
    x = re.findall("/([0-9\-]+)\.", url)
    if not x:
        return
    return base + ",".join(x[0].split("-"))

def book_price(task, rule):
    try:
        j = json.loads(task['text'])
        price = j['price'][0]['proPrice'] #if j['price'] else 0
    except:
        log_with_time("bad response: %s" % task['link'])
        return 
    return format_price([[str(task['qid']), str(price), task['stock']]])

def price_parser(task, rule):
    try:
        price = re.search("(?<=price\:)\d+\.\d+(?=\,)", task['text']).group()
    except:
        log_with_time("bad response: %r"%task['url'])
        return []
    ret = [(task['gid'], price, task['stock'])]
    fret = format_price(ret)
    return fret

def item_parser(task, rule):
    try:
        t = etree.HTML(task['text'])
        btn = t.xpath(rule)[0]
        stock = 0 if btn.attrib.get('disabled') else 1
    except:
        log_with_time("bad response: %s"%task['url'])
        return
    return [(task['gid'], stock)]

def price_parser(task, rule):
    try:
        items = jsonp_json(task["text"])
    except ValueError as e:
        log_with_time("price_parser: jsonp_json: %s" % task["text"])
        return
    d = {}
    for item in items:
        d[item["id"].split("_")[1]] =  item["p"]
    return [d]

def pager(task, rule):
    j = json.loads(task['text'])
    if not 'gpagecount' in j:
        log_with_time("bad response %r"%task['url'])
        return []
    code = re.search("(?<=code=)\d+(?=&)", task['url']).group()
    ret = []
    for i in range(1, j['gpagecount']+1):
        ret.append(gurl%(code,i))
    return ret

def stock_parser(task, rule):
    try:
        j = json.loads(task['text'])
        stock = 1 if j['havestock'] in ("true", "realstock") else 0
    except:
        log_with_time("bad response %s"%task['url'])
        return

    ret = [(itemurl % task['info'][0], str(task['info'][1]), stock)]
    fret = format_price(ret)
    return fret

def cats(url, res, rule):
    content = res["text"]
    try:
        t = etree.HTML(content)
    except:
        log_with_time("bad response %s" % content.decode("utf-8", "replace"))
        return
    ret = []
    for i in t.xpath(rule):
        ret.append(yougou + i)
    return ret