Python DB.open方法代码示例

# 需要导入模块: from kyotocabinet import DB [as 别名]
# 或者: from kyotocabinet.DB import open [as 别名]
def main():
    if len(sys.argv) < 3:
        sys.stderr.write('Usage: %s outdir textfile1 textfile2 ...\n'
                         % sys.argv[0])
        sys.exit(1)
    outdir = sys.argv[1]
    tfdb = DB()
    if not tfdb.open(os.path.join(outdir, 'tf.kch'),
                     DB.OWRITER | DB.OCREATE | DB.OTRUNCATE):
        sys.stderr.write('cannot open tfdb: %s\n' % str(tfdb.error))
        sys.exit(1)
    dfdb = DB()
    if not dfdb.open(os.path.join(outdir, 'df.kch'),
                     DB.OWRITER | DB.OCREATE | DB.OTRUNCATE):
        sys.stderr.write('cannot open dfdb: %s\n' % str(dfdb.error))
        sys.exit(1)
    tfidfdb = DB()
    if not tfidfdb.open(os.path.join(outdir, 'tfidf.kch'),
                     DB.OWRITER | DB.OCREATE | DB.OTRUNCATE):
        sys.stderr.write('cannot open tfidfdb: %s\n' % str(tfidfdb.error))
        sys.exit(1)

    print 'Count words ...'
    for i in range(len(sys.argv)-2):
        filename = sys.argv[i+2]
        print '(%d/%d) %s' % (i+1, len(sys.argv)-2, filename)
        count_words(tfdb, dfdb, filename)
    print 'Calculate TFIDF ...'
    save_tfidf(tfdb, dfdb, tfidfdb)

    tfdb.close()
    dfdb.close()
    tfidfdb.close()

# 需要导入模块: from kyotocabinet import DB [as 别名]
# 或者: from kyotocabinet.DB import open [as 别名]
 def dbOpen(name):
     db = DB()
     if self.create:
         # if not db.open(abspath(self.path) + '/' + name + ".kch",
         #         DB.OWRITER | DB.OCREATE | DB.OAUTOSYNC | DB.OAUTOTRAN):
         if not db.open(abspath(self.path) + "/" + name + ".kch", DB.OWRITER | DB.OCREATE):
             raise IOError("open error: " + str(db.error()))
         return db
     else:
         # if not db.open(abspath(self.path) + '/' + name + ".kch",
         #         DB.OWRITER | DB.OAUTOSYNC | DB.OAUTOTRAN):
         if not db.open(abspath(self.path) + "/" + name + ".kch", DB.OWRITER):
             raise IOError("open error: " + str(db.error()))
         return db

# 需要导入模块: from kyotocabinet import DB [as 别名]
# 或者: from kyotocabinet.DB import open [as 别名]
 def dbOpen(name):
     db = DB()
     dbpathname = abspath(self.path) + '/' + name + ".kch"
     if self.create:
         # if not db.open(abspath(self.path) + '/' + name + ".kch", 
         #         DB.OWRITER | DB.OCREATE | DB.OAUTOSYNC | DB.OAUTOTRAN):
         if not db.open(dbpathname, DB.OWRITER | DB.OCREATE):
             raise IOError("open error: %s %s" % (dbpathname, str(db.error())))  #pragma: NO COVER
         return db
     else:
         # if not db.open(abspath(self.path) + '/' + name + ".kch", 
         #         DB.OWRITER | DB.OAUTOSYNC | DB.OAUTOTRAN):
         if not db.open(dbpathname, DB.OWRITER):  #pragma: NO COVER
             raise IOError("open error: %s %s" % (dbpathname, str(db.error())))  #pragma: NO COVER
         return db

# 需要导入模块: from kyotocabinet import DB [as 别名]
# 或者: from kyotocabinet.DB import open [as 别名]
def get_post_num(post_num, db_file):
    item = None
    db = DB()
    if not db.open("{0}".format(db_file), DB.OREADER | DB.OCREATE):
        print "Could not open database."

    cur = db.cursor()
    cur.jump()
    i = 0
    while True:
        rec = cur.get(False)
        if not rec:
            break

        if i == post_num:
            item = rec

        cur.step()
        i = i + 1

    cur.disable()
    db.close()

    if item is not None:
        return loads(item[1])
    return dict()

# 需要导入模块: from kyotocabinet import DB [as 别名]
# 或者: from kyotocabinet.DB import open [as 别名]
def aggregate_by_hour(db_file):
    # Initialize the dict with each hour
    hours = {key: 0 for key in range(0,24)}
    db = DB()

    if not db.open("{0}".format(db_file), DB.OREADER | DB.OCREATE):
        print "Could not open database."

    cur = db.cursor()
    cur.jump_back()

    while True:
        rec = cur.get(False)
        if not rec:
            break

        loaded = loads(rec[1])
        unix = float(loaded['created_at'])
        time = datetime.fromtimestamp(unix)

        hours[time.hour] = hours[time.hour] + 1

        cur.step_back()
    cur.disable()
    db.close()

    hours = [{'name': "{}:00".format(key), 'data': [hours[key]]} for key in hours]
    return hours

# 需要导入模块: from kyotocabinet import DB [as 别名]
# 或者: from kyotocabinet.DB import open [as 别名]
def get_items(item_filter, db_file, page=0):
    item_iter = 0
    items = []
    db = DB()
    if not db.open("{0}".format(db_file), DB.OREADER | DB.OCREATE):
        print "Could not open database."

    cur = db.cursor()
    cur.jump_back()
    while len(items) < FILTER_MAX:
        rec = cur.get(False)
        if not rec:
            break

        if item_iter != (FILTER_MAX * page):
            if item_filter(rec):
                item_iter = item_iter + 1
            cur.step_back()
            continue

        if item_filter(rec):
            items.append(rec)

        cur.step_back()
    cur.disable()
    db.close()

    sorted_items = sorted(items, key=get_key, reverse=True)
    sorted_items_for_viewing = [loads(item[1]) for item in sorted_items]
    for item in sorted_items_for_viewing:
        if item['title'] is None or item['title'] == "":
            item['title'] = item['url']
    return sorted_items_for_viewing

# 需要导入模块: from kyotocabinet import DB [as 别名]
# 或者: from kyotocabinet.DB import open [as 别名]
    def decorated_function(*args, **kwargs):
        # Debug
        if not current_app.config['CACHE']:
            return f(*args, **kwargs)

        db = DB()
        db.open("/tmp/page_cache.kch")
        res = None
        fancy = hash("{}{}{}".format(db_meta_info()['count'], request.url, f.func_name))

        res = db.get(fancy)
        if not res:
            res = f(*args, **kwargs)
            db.set(fancy, res)

        db.close()
        return res

# 需要导入模块: from kyotocabinet import DB [as 别名]
# 或者: from kyotocabinet.DB import open [as 别名]
    def open(self, filename):
        db = DB()

        # self.db=DB(DB.GEXCEPTIONAL)
        if not db.open(filename, DB.OWRITER | DB.OCREATE | DB.ONOLOCK):
            raise IOError("open error: '" + str(self.db.error()) + "' on file:"
                          + filename)
        return db

# 需要导入模块: from kyotocabinet import DB [as 别名]
# 或者: from kyotocabinet.DB import open [as 别名]
def insert_item(url, person, db_file, submitted_title=''):
    mimetype = "application/json"
    db = DB()

    if not db.open("{0}".format(db_file),
        DB.OWRITER | DB.OCREATE):

        response = {}
        response['What happened?'] = "Couldn't open the damn database. Error: {0}".format(db.error())
        return Response(dumps(response), mimetype=mimetype)

    if is_url_in_db(db, url):
        return Response('{"What happened?": "Someone '\
            'tried to submit a duplicate URL."}',
            mimetype=mimetype)

    title = url
    summary = "~?~"
    try:
        thing = urlopen(url, timeout=10)
        soup = BeautifulSoup(thing)
        title = ''
        if len(submitted_title) > 0:
            title = submitted_title
        else:
            title = soup.title.string
        # Do some dumb summarizing if we can
        func = lambda a,v: a + " " + v.strip()
        visible_stuff = filter(visible, soup.findAll(text=True))
        summary = reduce(func, visible_stuff, "")[:900] + "..."
    except:
        pass
        #return Response('{"What happened?": '\
        #    'I dunno bs4 messed up somehow."}',
        #    mimetype=mimetype)

    created_at = int(mktime(datetime.now().utctimetuple()))

    is_image = url.lower().endswith(("jpg", "jpeg", "gif", "png"))
    thumbnail = gen_thumbnail_for_url(url, str(created_at))

    record = {
        "created_at": created_at,
        "title": title,
        "url": url,
        "person": person,
        "summary": summary,
        "person_color": PERSON_COLORS[random.randint(0, len(PERSON_COLORS)-1)],
        "is_image": is_image,
        "thumbnail": thumbnail
    }
    db.set(created_at, dumps(record))
    db.close()

    return Response('{"What happened?": "MUDADA"}',
        mimetype=mimetype)

# 需要导入模块: from kyotocabinet import DB [as 别名]
# 或者: from kyotocabinet.DB import open [as 别名]
def top_things(db_file):
    urls = {}
    people = {}
    graph = {}

    db = DB()

    if not db.open("{0}".format(db_file), DB.OREADER | DB.OCREATE):
        print "Could not open database. (Top things)"

    cur = db.cursor()
    cur.jump_back()
    while True:
        rec = cur.get(False)
        if not rec:
            break

        loaded_rec = loads(rec[1])
        split = get_domain(loaded_rec)

        if urls.get(split, False) == False:
            urls[split] = 1
        else:
            urls[split] = urls[split] + 1

        person = loaded_rec['person']
        if people.get(person, False) == False:
            people[person] = 1
        else:
            people[person] = people[person] + 1

        if split is not None and split is not "" and \
            person is not None and person is not "":
            # Build a crazy relational graph out of my nosql data
            if graph.get(split, False) == False:
                graph[split] = {"is_person": False, "data": [person], "linked_to_count": 1}
            elif person not in graph[split]:
                graph[split]["data"].append(person)
                graph[split]["linked_to_count"] = graph[split]["linked_to_count"] + 1

            if graph.get(person, False) == False:
                graph[person] = {"is_person": True, "data": [split]}
            elif split not in graph[person]:
                graph[person]["data"].append(split)

        cur.step_back()
    cur.disable()
    db.close()

    def get_one(x):
        return x[1]

    return (sorted(urls.items(), key=get_one, reverse=True),
            sorted(people.items(), key=get_one, reverse=True),
            graph)

# 需要导入模块: from kyotocabinet import DB [as 别名]
# 或者: from kyotocabinet.DB import open [as 别名]
def get_post_by_date(key, db_file):
    item = None
    db = DB()
    if not db.open("{0}".format(db_file), DB.OREADER | DB.OCREATE):
        print "Could not open database."
    item = db.get(key)

    db.close()
    if item is not None:
        return loads(item)
    return dict()

# 需要导入模块: from kyotocabinet import DB [as 别名]
# 或者: from kyotocabinet.DB import open [as 别名]
def db_meta_info():
    meta = {}
    db = DB()
    db_file = current_app.config['DB_FILE']
    if not db.open("{0}".format(db_file), DB.OREADER):
        print "Could not open database (meta info)."
    meta["size"] = db.size()
    meta["count"] = db.count()
    db.close()

    return meta

# 需要导入模块: from kyotocabinet import DB [as 别名]
# 或者: from kyotocabinet.DB import open [as 别名]
def purge(domain, genid):
    if request.remote_addr not in settings.ALLOW:
        return text_response("Not permitted.\n", 403)

    db = DB()

    if not db.open(settings.GENID_DATABASE, DB.OWRITER | DB.OCREATE):
        return text_response("Failed to purge: cannot open database.\n", 501)

    set_ok = db.set(domain, genid)
    db.close()

    if not set_ok:
        return text_response("Failed to purge: cannot set genid.\n", 501)
    else:
        return text_response("Purged <%s>\n" % (domain,))

# 需要导入模块: from kyotocabinet import DB [as 别名]
# 或者: from kyotocabinet.DB import open [as 别名]
def get_items_last_X_days(db_file, X, munge=True):
    dates = {}
    db = DB()
    if not db.open("{0}".format(db_file), DB.OREADER | DB.OCREATE):
        print "Could not open database."

    X_days_ago = datetime.now() - timedelta(days=X)

    cur = db.cursor()
    cur.jump_back()
    while True:
        rec = cur.get(False)
        if not rec:
            break

        loaded = loads(rec[1])
        unix = float(loaded['created_at'])
        time = datetime.fromtimestamp(unix)

        if time > X_days_ago:
            if munge:
                date_obj = date(year=time.year, month=time.month, day=time.day)
            else:
                date_obj = time
            # Javascript expects Date.UTC to spit out dates of a certain
            # length.
            day_unix = int(mktime(date_obj.timetuple()))*1000
            if dates.get(day_unix, False) == False:
                dates[day_unix] = {loaded["person"]: 1}
            else:
                relevant_dict = dates[day_unix]

                if relevant_dict.get(loaded["person"], False) == False:
                    relevant_dict[loaded["person"]] = 1
                else:
                    relevant_dict[loaded["person"]] = relevant_dict[loaded["person"]] + 1
        else:
            break;

        cur.step_back()
    cur.disable()
    db.close()

    return dates

# 需要导入模块: from kyotocabinet import DB [as 别名]
# 或者: from kyotocabinet.DB import open [as 别名]
def get_last_items(db_file, pages=1):
    items = []
    db = DB()
    if not db.open("{0}".format(db_file), DB.OREADER | DB.OCREATE):
        print "Could not open database."

    cur = db.cursor()
    cur.jump_back()
    while len(items) < (pages * FILTER_MAX):
        rec = cur.get(False)
        if not rec:
            break

        items.append(rec)
        cur.step_back()
    cur.disable()
    db.close()

    return items