Python Utils.getConfig方法代码示例

# 需要导入模块: from utils import Utils [as 别名]
# 或者: from utils.Utils import getConfig [as 别名]
    def updateRegularProducts(self, productRegular):
        print 'Updating prices for regular products...'
        try:
            con = mdb.connect(Utils.getConfig()['host'], Utils.getConfig()['user'],
                              Utils.getConfig()['passwd'], Utils.getConfig()['dbname'])
            cur = con.cursor(mdb.cursors.DictCursor)
            # select the most recent records for each product
            start = time.time()
            # ids = ''
            # for prod in productRegular:
            #     ids = ids + str(prod['id']) + ','
            # ids = ids[:-1]
            # cur.execute("SELECT p1.* FROM product_price p1 LEFT JOIN product_price p2 \
            #             ON (p1.id = p2.id AND p1.date < p2.date) WHERE p2.date IS NULL AND p1.id IN ({})".format(ids))
            cur.execute("SELECT * FROM product_price")
            productRegularDB = cur.fetchall()
            end = time.time()
            print "Query time: {}".format(end - start)
            # sort by date (descending) and by product ID
            productRegularDB = sorted(productRegularDB, key=lambda k: (k['date']), reverse=True)
            productRegularDB = sorted(productRegularDB, key=lambda k: (k['id']))
            Utils.deleteDuplicates(productRegularDB)
            idArray = Utils.buildArray(productRegularDB)

            for product in productRegular:
                index = Utils.binarySearch(idArray, product['id'])
                # insert a new product record if it hasn't occurred before or the price has changed 
                if index == -1 or productRegularDB[index]['price'] != product['price']:
                    cur.execute("INSERT INTO product_price (id, date, price, category, url) \
                    VALUES ({}, \"{}\", {}, {}, \"{}\")".format(product['id'], product['date'], product['price'],
                                                                product['category'], product['url']))
        except mdb.Error, e:
            print "Error %d: %s" % (e.args[0], e.args[1])
            sys.exit(1)

# 需要导入模块: from utils import Utils [as 别名]
# 或者: from utils.Utils import getConfig [as 别名]
 def parseFile(self):
     print 'Parsing product file...'
     productsRegular = []
     productsPromo = []
     # open products file and process items
     with open(Utils.getConfig()['productFile']) as json_file:
         json_data = json.load(json_file)
     for item in json_data:
         product = {}
         # not available products are omitted
         # if item['avail'] == self.PROD_NOT_AVAIL:
         #     continue
         product['id'] = long(item['id'])
         product['price'] = float(item['price'])
         product['category'] = item['cat']
         # product['name'] = data['name'].encode('utf-8')
         product['url'] = item['url'].encode('utf-8')
         # in case of regular product add it to separate table
         if item.get('oldPr', None) is None:
             product['date'] = self.dateTime[:10]
             productsRegular.append(product)
         else:
             product['old_price'] = item['oldPr']
             product['discount'] = item['disc']
             # product['description'] = data['descr'].encode('utf-8')
             product['url'] = item['url'].encode('utf-8')
             productsPromo.append(product)
     self.updateRegularProducts(productsRegular)
     self.updatePromoProducts(productsPromo)
     Utils.renameFile(Utils.getConfig()['productFile'])

# 需要导入模块: from utils import Utils [as 别名]
# 或者: from utils.Utils import getConfig [as 别名]
 def parseFile(self):
     print "Parsing product file..."
     productsRegular = []
     productsPromo = []
     # open products file and process items
     with open(Utils.getConfig()["productFile"]) as json_file:
         json_data = json.load(json_file)
     for item in json_data:
         product = {}
         # not available products are omitted
         # if item['avail'] == self.PROD_NOT_AVAIL:
         #     continue
         try:
             product["id"] = long(item["id"])
             product["price"] = float(item["price"])
             product["category"] = item["cat"]
             # product['name'] = data['name'].encode('utf-8')
             product["url"] = item["url"].encode("utf-8")
             # in case of regular product add it to separate table
             if item.get("oldPr", None) is None:
                 product["date"] = self.dateTime[:10]
                 productsRegular.append(product)
             else:
                 product["old_price"] = item["oldPr"]
                 product["discount"] = item["disc"]
                 # product['description'] = data['descr'].encode('utf-8')
                 product["url"] = item["url"].encode("utf-8")
                 productsPromo.append(product)
         except:
             pass
     self.updateRegularProducts(productsRegular)
     self.updatePromoProducts(productsPromo)
     Utils.renameFile(Utils.getConfig()["productFile"])

# 需要导入模块: from utils import Utils [as 别名]
# 或者: from utils.Utils import getConfig [as 别名]
 def getProducts(self):
     with open(Utils.getConfig()['subcatFile']) as json_file:
         jsonData = json.load(json_file)
     self.getProductsByCat(jsonData)
     self.items = sorted(self.items, key=lambda k: (k['id']))
     Utils.saveJsonFile(Utils.getConfig()['subcatFile'], jsonData)
     Utils.deleteDuplicates(self.items)
     Utils.saveJsonFile(Utils.getConfig()['productFile'], self.items)

# 需要导入模块: from utils import Utils [as 别名]
# 或者: from utils.Utils import getConfig [as 别名]
 def finish(self):
     print("Więcej informacji: https://github.com/thof/decapromolist#decapromolist" + self.SPACES, file=self.mdFile)
     print(
         "PayPal: [email protected]_ (w razie gdyby ktoś chciał wspomóc projekt)" + self.SPACES,
         file=self.mdFile,
     )
     self.mdFile.close()
     Utils.renameFile(Utils.getConfig()["decapromolistFile"])
     Utils.saveJsonFile(Utils.getConfig()["decapromolistFile"], self.products)

# 需要导入模块: from utils import Utils [as 别名]
# 或者: from utils.Utils import getConfig [as 别名]
    def __init__(self, model, flickr, parent=None):
        super(UploadFrame, self).__init__(parent)
        self.model = model
        self.flickr = flickr

        self.photos = []
        self.directory = None
        self.check_thread = None

        #self.setFixedSize(450,120)
        main_layout = QtGui.QGridLayout()
        
        self.edit_directory = QtGui.QLineEdit()
        self.edit_directory.setReadOnly(True)    
        self.edit_directory.setText(Utils.getConfig('directory'))
        self.edit_directory.setMaximumWidth(320)
        self.edit_directory.setMinimumWidth(320)
        if Utils.getConfig('directory'):
            self.directory = Utils.getConfig('directory')
        #button_directory.setMaximumWidth(290)
        self.button_browse = QtGui.QPushButton(self.tr("browse"))
        self.button_browse.setMaximumWidth(80)
        self.button_sync = QtGui.QPushButton(self.tr("sync"))
        self.button_sync.setMaximumWidth(80)
        
        self.button_browse.clicked.connect(self.browse)
        self.button_sync.clicked.connect(self.startUploadThread)

        self.button_more = QtGui.QCheckBox(self.tr('more'))

        extension = QtGui.QWidget()

        self.button_delete_db = QtGui.QPushButton("clean")
        self.button_delete_db.clicked.connect(self.deleteDB)

        button_delete_auth = QtGui.QPushButton("unlink")
        button_delete_auth.clicked.connect(self.deleteAuth)

        self.button_more.toggled.connect(extension.setVisible)

        extensionLayout = QtGui.QHBoxLayout()
        extensionLayout.setContentsMargins(0, 0, 0, 0)
        extensionLayout.addWidget(self.button_delete_db)
        extensionLayout.addWidget(button_delete_auth)
        extension.setLayout(extensionLayout)

        main_layout.setSizeConstraint(QtGui.QLayout.SetFixedSize)
        main_layout.addWidget(self.edit_directory, 0, 0, 1, 1, alignment=QtCore.Qt.AlignLeft)
        main_layout.addWidget(self.button_browse, 0, 1, 1, 1, alignment=QtCore.Qt.AlignLeft)
        main_layout.addWidget(self.button_sync, 1, 0, 1, 1, alignment=QtCore.Qt.AlignLeft)
        main_layout.addWidget(self.button_more, 2, 0, 1, 2, alignment=QtCore.Qt.AlignRight)
        main_layout.addWidget(extension, 3, 0, 1, 2, alignment=QtCore.Qt.AlignLeft)
        
        self.setLayout(main_layout)
        extension.hide()

# 需要导入模块: from utils import Utils [as 别名]
# 或者: from utils.Utils import getConfig [as 别名]
 def getCategories(self):
     categories = []
     catUrl = []
     content = urllib2.urlopen(Utils.getConfig()["siteURL"]).read()
     response = html.fromstring(content)
     for cat in response.xpath("//li/@primarycategoryid"):
         if cat not in categories:
             categories.append(cat)
     for cat in categories:
         url = "{}/pl/getSubNavigationMenu?primaryCategoryId={}".format(Utils.getConfig()["siteURL"], cat)
         catUrl.append(url)
     return catUrl

# 需要导入模块: from utils import Utils [as 别名]
# 或者: from utils.Utils import getConfig [as 别名]
 def getPrevProcDate(self):
     # get date of the previous processing
     try:
         con = mdb.connect(Utils.getConfig()['host'], Utils.getConfig()['user'],
                           Utils.getConfig()['passwd'], Utils.getConfig()['dbname'])
         cur = con.cursor(mdb.cursors.DictCursor)
         cur.execute("SELECT last_date FROM product_promo WHERE last_date < \"{}\" ORDER BY last_date DESC LIMIT 1"
                     .format(self.dateTime))
         row = cur.fetchone()
         self.datePrevProcFormatted = row['last_date'].strftime('%d.%m.%Y')
     except mdb.Error, e:
         print("Error %d: %s" % (e.args[0], e.args[1]))
         sys.exit(1)

# 需要导入模块: from utils import Utils [as 别名]
# 或者: from utils.Utils import getConfig [as 别名]
 def getProductsByCat(self, subcat):
     for index, cat in enumerate(subcat):
         #print cat['url']
         self.counter = 0
         page = 0
         while True:
             page += 1
             url = "http://www.decathlon.pl/pl/getAjaxListProductNextPage?uriPage=/C-{}/I-Page{}_40".format(cat['subId'], page)
             #print url
             try:
                 self.parse(cat['subId'], url)
             except urllib2.HTTPError as httpError:
                 print httpError
                 print url
                 if str(httpError.code)[0] == '5':
                     self.parse(cat['subId'], url)
                 else:
                     break
             except IndexError:
                 if page == 1:
                     urlCat = "{}/pl/getSubNavigationMenu?primaryCategoryId={}".format(
                         Utils.getConfig()['siteURL'], cat['subId'])
                     print "*** Let's try to seek deeper {}".format(url)
                     dataCat = GetSubcategories.getThirdLevelCat([urlCat])
                     subcat[index+1:index+1] = dataCat
                     self.getProductsByCat(dataCat)
                 break
         print "{} {}".format(cat['url'].encode('utf-8'), self.counter)

# 需要导入模块: from utils import Utils [as 别名]
# 或者: from utils.Utils import getConfig [as 别名]
 def commitChanges(self):
     os.chdir(Utils.getConfig()['decapromolistDir'])
     print call("pwd", shell=True)
     print call("git add decapromolist/", shell=True)
     print call("git add category/", shell=True)
     call("git commit -m 'delta {}-{}'".format(PrepareResults.datePrevProcFormatted,
                                               PrepareResults.dateFormatted), shell=True)
     call("git push -u origin master", shell=True)

# 需要导入模块: from utils import Utils [as 别名]
# 或者: from utils.Utils import getConfig [as 别名]
 def getPrevProcDate(self):
     # get date of the previous processing
     try:
         con = mdb.connect(
             Utils.getConfig()["host"],
             Utils.getConfig()["user"],
             Utils.getConfig()["passwd"],
             Utils.getConfig()["dbname"],
         )
         cur = con.cursor(mdb.cursors.DictCursor)
         cur.execute(
             'SELECT last_date FROM product_promo WHERE last_date < "{}" ORDER BY last_date DESC LIMIT 1'.format(
                 self.dateTime
             )
         )
         row = cur.fetchone()
         self.datePrevProcFormatted = row["last_date"].strftime("%d.%m.%Y")
     except mdb.Error, e:
         print("Error %d: %s" % (e.args[0], e.args[1]))
         sys.exit(1)

# 需要导入模块: from utils import Utils [as 别名]
# 或者: from utils.Utils import getConfig [as 别名]
 def getThirdLevelCat(catUrl):
     dataCat = []
     for url in catUrl:
         content = urllib2.urlopen(url).read()
         jsonData = json.loads(content)
         for cat in jsonData["category"]["categories"]:
             data = {
                 "id": int(jsonData["category"]["id"]),
                 "name": jsonData["category"]["label"],
                 "subId": int(cat["id"]),
                 "subName": cat["label"],
             }
             if cat["uri"].find(Utils.getConfig()["siteURL"]) == -1:
                 data["url"] = Utils.getConfig()["siteURL"] + cat["uri"]
             else:
                 data["url"] = cat["uri"]
                 data["subId"] = int(
                     cat["uri"][cat["uri"].find("C-") + 2 : cat["uri"].find("-", cat["uri"].find("C-") + 2)]
                 )
             dataCat.append(data)
     return dataCat

# 需要导入模块: from utils import Utils [as 别名]
# 或者: from utils.Utils import getConfig [as 别名]
 def getProducts(self):
     with open(Utils.getConfig()['subcatFile']) as json_file:
         self.jsonData = json.load(json_file)
     for cat in self.jsonData:
         print cat['url']
         page = 0
         while True:
             page += 1
             url = cat['url'].encode('utf-8') + "/I-Page{}_40".format(page)
             print url
             try:
                 self.parse(cat['subId'], url)
             except urllib2.HTTPError as httpError:
                 print httpError
                 if str(httpError.code)[0] == '5':
                     self.parse(cat['subId'], url)
                 else:
                     break
             except IndexError:
                 break
     self.items = sorted(self.items, key=lambda k: (k['id']))
     Utils.deleteDuplicates(self.items)
     Utils.saveJsonFile(Utils.getConfig()['productFile'], self.items)

# 需要导入模块: from utils import Utils [as 别名]
# 或者: from utils.Utils import getConfig [as 别名]
 def getSubcategories(self, catUrl):
     dataCat = []
     for url in catUrl:
         content = urllib2.urlopen(url).read()
         jsonData = json.loads(content)
         for cat in jsonData["category"]["categories"]:
             for subcat in cat["categories"]:
                 data = {
                     "id": int(cat["id"]),
                     "name": cat["label"],
                     "subId": int(subcat["id"]),
                     "subName": subcat["label"],
                     "url": Utils.getConfig()["siteURL"] + subcat["uri"],
                 }
                 dataCat.append(data)
     return dataCat

# 需要导入模块: from utils import Utils [as 别名]
# 或者: from utils.Utils import getConfig [as 别名]
    def prepareRegularList(self):
        try:
            con = mdb.connect(Utils.getConfig()['host'], Utils.getConfig()['user'],
                              Utils.getConfig()['passwd'], Utils.getConfig()['dbname'])
            cur = con.cursor(mdb.cursors.DictCursor)
            #             cur.execute('SELECT p1.* FROM product_price p1 LEFT JOIN product_price p2 \
            #                         ON (p1.id = p2.id AND p1.price > p2.price) WHERE p2.price IS NULL')
            #             productPriceLowest = cur.fetchall()
            # get all products from DB collected today
            cur.execute("SELECT * FROM product_price WHERE date = \"{}\"".format(self.dateTime[:10]))
            productCurrentPrice = cur.fetchall()

            ids = ''
            for prod in productCurrentPrice:
                ids = ids + str(prod['id']) + ','
            ids = ids[:-1]
            # get all products from DB for IDs fetched in the previous query with date different from today
            cur.execute("SELECT * FROM product_price WHERE date != \"{}\" AND id IN ({})".format(self.dateTime[:10], ids))
            productPrevPrice = cur.fetchall()

            print("\n\nLista przecenionych produktów (delta {}-{}):\n".format(self.datePrevProcFormatted,
                                                                            self.dateFormatted), file=self.mdFile)

            productFinal = []
            # looking for products for which current price is lower than the last one
            for prod in productCurrentPrice:
                prodLast = next((row for row in productPrevPrice if row['id'] == prod['id']), None)
                if prodLast is not None and prod['price'] < prodLast['price']:
                    prod['discount'] = int((1 - prod['price'] / prodLast['price']) * 100)
                    # don't take into account when the discount is 10% or less
                    if prod['discount'] < 10:
                        continue
                    prod['prev_price'] = prodLast['price']
                    productFinal.append(prod)
            productFinal = sorted(productFinal, key=lambda k: (k['discount'], k['price']), reverse=True)

            # prepare information to be printed in a readable form
            for prod in productFinal:
                product = {}
                url = 'http://www.decathlon.pl' + prod['url']
                content = urllib2.urlopen(url).read()
                response = html.fromstring(content)

                name = response.xpath('//span[@id="productName"]')[0].text
                if name == name.upper():
                    name = name.title().encode('utf-8')
                else:
                    name = name.encode('utf-8')

                text = "[{}](http://www.decathlon.pl{}) {}->{} ({}%)".format(name, prod['url'], prod['prev_price'],
                                                                             prod['price'], prod['discount'])

                product['nm'] = "<a href=\"http://www.decathlon.pl"+prod['url']+"\">"+name+"</a>"
                product['rl'] = prod['url']
                product['pr'] = prod['price']
                product['pp'] = prod['prev_price']
                product['op'] = prod['prev_price']
                product['or'] = "Obniżka"
                cat = next((row for row in self.subcatData if row['subId'] == prod['category']), None)
                if cat is not None:
                    product['sc'] = cat['name'].encode('utf-8') + "->" + cat['subName'].encode('utf-8')
                # additional check to be sure that the current price is the lowest to this day
                # (checking "price history")
                prodLowestPrice = next(
                        (row for row in productPrevPrice if row['id'] == prod['id'] and row['price'] < prod['price']),
                        None)
                if prodLowestPrice is not None:
                    text = text + " [Regularna cena była niższa {} w dn. {}]".format(prodLowestPrice['price'],
                                                                                     prodLowestPrice['date'])
                    product['rp'] = prodLowestPrice['price']
                    product['rd'] = prodLowestPrice['date'].strftime("%d.%m.%Y")
                print(text+self.SPACES, file=self.mdFile)
                self.products.append(product)
            print("", file=self.mdFile)
        except mdb.Error, e:
            print("Error %d: %s" % (e.args[0], e.args[1]))
            sys.exit(1)