Python Product.updated方法代码示例

# 需要导入模块: from models import Product [as 别名]
# 或者: from models.Product import updated [as 别名]
    def crawl_listing(self, url, ctx='', **kwargs):
        res = requests.get(url)
        res.raise_for_status()
        tree = lxml.html.fromstring(res.content)

        category = Category.objects(key=kwargs.get('key')).first()
        if not category:
            common_failed.send(sender=ctx, url=url, reason='category %s not found in db' % kwargs.get('key'))
            return

        product_nodes = tree.cssselect('div#searchResults a')
        for product_node in product_nodes:
            price = None; listprice = None
            price = product_node.cssselect('.price-6pm')[0].text
            listprice_node = product_node.cssselect('.discount')
            listprice = ''.join(listprice_node[0].xpath('text()')) if listprice_node else None

            # eliminate products of no discountIndexError:
            if price is None or listprice is None:
                # common_failed.send(sender=ctx, url=url, \
                #     reason='listing product %s.%s cannot crawl price info -> %s / %s' % (key, title, price, listprice))
                continue

            key = product_node.get('data-product-id')
            if not key:
                common_failed.send(sender=ctx, url=url, reason='listing product has no key')
                continue

            combine_url = product_node.get('href')
            key = '%s_%s' % (key, combine_url.split('/')[-1])
            match = re.search(r'https?://.+', combine_url)
            if not match:
                combine_url = '%s%s' % (HOST, combine_url)

            brand = product_node.cssselect('.brandName')[0].text.strip()
            title = product_node.cssselect('.productName')[0].text.strip()

            is_new = False; is_updated = False
            product = Product.objects(key=key).first()
            if not product:
                is_new = True
                product = Product(key=key)
                product.updated = False
                product.event_type = False

            if title and title != product.title:
                product.title = title
                is_updated = True

            if brand and brand != product.brand:
                product.brand = brand
                is_updated = True

            if combine_url and combine_url != product.combine_url:
                product.combine_url = combine_url
                is_updated = True

            if price and price != product.price:
                product.price = price
                is_updated = True

            if listprice and listprice != product.listprice:
                product.listprice = listprice
                is_updated = True

            if category.cats and set(category.cats).difference(product.dept):
                product.dept = list(set(category.cats) | set(product.dept or []))
                is_updated = True

            if category.key not in product.category_key:
                product.category_key.append(category.key)
                is_updated = True

            if is_updated:
                product.list_update_time = datetime.utcnow()
            
            # To pick the product which fit our needs, such as a certain discount, brand, dept etc.
            selected = Picker(site='6pm').pick(product)
            if not selected:
                continue

            product.hit_time = datetime.utcnow()
            product.save()

            common_saved.send(sender=ctx, obj_type='Product', key=product.key, url=product.combine_url, \
                is_new=is_new, is_updated=((not is_new) and is_updated) )


            print product.key; print product.brand; print product.title; \
            print product.price, ' / ', product.listprice; print product.combine_url; \
            print product.dept; print

        # Go to the next page to keep on crawling.
        next_page = None
        page_node = tree.cssselect('div.pagination')
        if not page_node:
            return

        last_node =page_node[0].cssselect('.last')
        if last_node:
#.........这里部分代码省略.........

# 需要导入模块: from models import Product [as 别名]
# 或者: from models.Product import updated [as 别名]
    def crawl_product(self, url, casin, ctx='', **kwargs):
        r = req.get(url)
        data = re.compile(r'parse_asin_\w+\((.*)\);$').search(r.text).group(1)
        data = json.loads(data)

        image_urls = []
        for i in data['detailJSON']['main']['altviews']:
            if i['zoomImage'] not in image_urls:
                image_urls.append(i['zoomImage'])

        if not image_urls:
            for i in data['detailJSON']['asins']:
                if i['asin'] == casin:
                    for j in i['altviews']:
                        if j['zoomImage'] not in image_urls:
                            image_urls.append(j['zoomImage'])
                    break

        asin = data['detailJSON']['asin']
        summary = data['productDescription']['shortProdDesc']
        if data['productDescription']['bullets']:
            list_info = [i.replace('"', '"').replace(''', '\'') for i in data['productDescription']['bullets'][0]['bulletsList']]
        else:
            list_info = []
        brand = data['detailJSON']['brand']
        returned = data['detailJSON']['returnPolicy']
#        if 'intlShippable' in data['detailJSON']:
#            shipping = 'international shipping' if data['detailJSON']['intlShippable'] == 1 else 'no international shipping'
#        elif 'choices' in data['detailJSON']:
#            for i in data['detailJSON']['choices']:
#                if i['asin'] == casin:
#                    shipping = 'international shipping' if i['intlShippable'] == 1 else 'no international shipping'
#                    break
#        shipping = shipping if shipping else ''

        video = ''
        for p in data['detailJSON']['asins']:
            if p['asin'] == casin:
                video = p['videos'][0]['url'] if p['videos'] else ''
                break

        is_new, is_updated = False, False
        product = Product.objects(key=casin).first()
        if not product:
            is_new = True
            product = Product(key=casin)
        product.summary = summary
        product.list_info = list_info
        product.brand = brand
        product.shipping = 'FAST, FREE SHIPPING, FREE RETURN SHIPPING in the U.S.'
        product.returned = returned
        product.video = video
        product.image_urls = image_urls
        product.full_update_time = datetime.utcnow()

        if product.updated == False:
            product.updated = True
            ready = True
        else: ready = False
        product.save()
        common_saved.send(sender=ctx, obj_type='Product', key=casin, url=url, is_new=is_new, is_updated=is_updated, ready=ready)

# 需要导入模块: from models import Product [as 别名]
# 或者: from models.Product import updated [as 别名]
    def _parse_product(self, event_id, asins, cAsins, prefix_url, product_data, ctx):
        """ no video info, list_info, summary

        :param event_id: this product belongs to the event's id
        :param asins: all asins info in this event
        :param cAsins: all casins info in this event
        :param prefix_url: image and js prefix_url, probably 'http://z-ecx.images-amazon.com/images/I/'
        :param product_data: product data in this product
        """
        asin = product_data['asin']
        casin = product_data['cAsin']
        title = product_data['title'].encode('utf-8') # color is in title
#        image_urls = [product_data['image']] + product_data['altImages'] # one picture, altImages is []
        if 'listPrice' in product_data:
            listprice = product_data['listPrice']['display'] # or 'amount', if isRange: True, don't know what 'amount' will be
        else: listprice = ''
        price = product_data['ourPrice']['display']
        sizes = []
        if product_data['teenagers']: # no size it is {}
            for k, v in product_data['teenagers'].iteritems():
                if v['size'] not in sizes: sizes.append(v['size'])
        # tag is not precision. e.g. a bag is in shoes
        # tag = product_data['productGL'] if 'productGL' in product_data else '' # 'apparel', 'home', 'jewelry', ''

        soldout_link = 'http://www.myhabit.com/request/getBuyableAsinInfo?asin={0}&saleId={1}&flavor=parent&sid=177-4704555-7345351'.format(asin, event_id)
        # one soldout link contains this asin's all color.
        ret = req.get(soldout_link)
        jsdata = json.loads(ret.content)
        key_list = sorted(jsdata['buyableAsin'].keys())
        len_sizes = len(sizes)
        soldout = False
        if len_sizes == 0:
            if jsdata['buyableAsin'][casin]['stats']['remaining']['claimed'] == 0:
                soldout = True
            else:
                soldout = False

        else: # more than one size.
            if 'asin' in key_list: key_list.remove('asin')
            if 'privateSaleID' in key_list: key_list.remove('privateSaleID')

            count = 0
            for l in key_list:
                if l == casin or (count > 0 and count < len_sizes):
                    count += 1
                    if jsdata['buyableAsin'][l]['stats']['remaining']['claimed'] == 0:
                        soldout = True
                    else:
                        soldout = False
                        break
        
#        if casin in cAsins and 'soldOut' in cAsins[casin] and cAsins[casin]['soldOut'] == 1:
#            soldout = True
#        else: soldout = False
        jslink = prefix_url + asins[asin]['url'] if asin in asins else ''
        combine_url = 'http://www.myhabit.com/homepage#page=d&sale={0}&asin={1}&cAsin={2}'.format(event_id, asin, casin)

        is_new, is_updated = False, False
        product = Product.objects(key=casin).first()
        if not product:
            is_new = True
            product = Product(key=casin)
            product.combine_url = combine_url
            product.asin = asin
            product.title = title
#            product.image_urls = image_urls
            product.listprice = listprice
            product.price = price
            product.sizes = sizes
            product.soldout = soldout
            product.updated = False
        else:
            if soldout and product.soldout != soldout:
                product.soldout = True
                is_updated = True
                product.update_history.update({ 'soldout': datetime.utcnow() })
            if product.title != title:
                product.title = title
                product.update_history.update({ 'title': datetime.utcnow() })
            if product.combine_url != combine_url:
                product.combine_url = combine_url
                product.update_history.update({ 'combine_url': datetime.utcnow() })
            if product.listprice != listprice:
                product.listprice = listprice
                product.update_history.update({ 'listprice': datetime.utcnow() })
            if product.price != price:
                product.price = price
                product.update_history.update({ 'price': datetime.utcnow() })

        if event_id not in product.event_id: product.event_id.append(event_id)
        product.jslink = jslink
        product.list_update_time = datetime.utcnow()
        product.save()
        common_saved.send(sender=ctx, obj_type='Product', key=casin, url=product.combine_url, is_new=is_new, is_updated=is_updated)
        return casin