Python OpenLibrary.query方法代码示例

# 需要导入模块: from openlibrary.api import OpenLibrary [as 别名]
# 或者: from openlibrary.api.OpenLibrary import query [as 别名]
def main(server):
    ol = OpenLibrary(server)
    ol.autologin()

    volumes = ol.query({'type': '/type/volume', 'limit': False, '*': None, 'edition': {'*': None}})

    volumes = dict((v['key'], v) for v in volumes)
    editions = dict((v['edition']['key'], v['edition']) for v in volumes.values() if v['edition'])

    def make_volume(v):
        d = {}
        v.pop('edition')
        v['type'] = {'key': '/type/volume'}
        for k in ['type', 'ia_id', 'volume_number']:
            if k in v:
                d[k] = v[k]
        return d

    for e in editions.values():
        e['volumes'] = []

    for v in volumes.values():
        if v.get('edition'):
            e = editions[v.get('edition')['key']]
            e['volumes'].append(make_volume(v))

    for e in editions.values():
        e['volumes'] = sorted(e['volumes'], key=lambda v: v['volume_number'])

    print 'linking volumes to %d editions' % len(editions)
    ol.save_many(editions.values(), 'link volumes')

# 需要导入模块: from openlibrary.api import OpenLibrary [as 别名]
# 或者: from openlibrary.api.OpenLibrary import query [as 别名]
def main():
    global options
    options, args = parse_options()
    ol = OpenLibrary(options.server)

    for pattern in args:
        docs = ol.query({"key~": pattern, "*": None}, limit=1000)
        for doc in marshal(docs):
            if doc['type']['key'] == '/type/template':
                write(make_path(doc), get_value(doc, 'body'))
            elif doc['type']['key'] == '/type/template':
                write(make_path(doc), get_value(doc, 'macro'))
            else:
                delete(make_path(doc))

# 需要导入模块: from openlibrary.api import OpenLibrary [as 别名]
# 或者: from openlibrary.api.OpenLibrary import query [as 别名]
def main():
    ol = OpenLibrary()
    ol.autologin()

    plugin = sys.argv[1]

    all_docs = []

    for pattern in sys.argv[2:]:
        docs = ol.query({"key~": pattern, "*": None}, limit=1000)
        all_docs.extend(docs)

    for doc in all_docs:
        doc['plugin'] = plugin

    print ol.save_many(all_docs, comment="Marked as part of %s plugin." % plugin)

# 需要导入模块: from openlibrary.api import OpenLibrary [as 别名]
# 或者: from openlibrary.api.OpenLibrary import query [as 别名]
def main():
    global options
    options, args = parse_options()
    ol = OpenLibrary(options.server)

    for pattern in args:
        docs = ol.query({"key~": pattern, "*": None}, limit=1000)
        for doc in marshal(docs):
            # Anand: special care to ignore bad documents in the database.
            if "--duplicate" in doc['key']:
                continue

            if doc['type']['key'] == '/type/template':
                write(make_path(doc), get_value(doc, 'body'))
            elif doc['type']['key'] == '/type/macro':
                write(make_path(doc), get_value(doc, 'macro'))
            else:
                delete(make_path(doc))

# 需要导入模块: from openlibrary.api import OpenLibrary [as 别名]
# 或者: from openlibrary.api.OpenLibrary import query [as 别名]
def main(server):
    ol = OpenLibrary(server)
    ol.autologin()

    scan_records = ol.query(type='/type/scan_record', limit=False, edition={'*': None})
    editions = (r['edition'] for r in scan_records)

    # process 1000 editions at a time.
    while True:
        chunk = take(1000, editions)
        if not chunk:
            break

        print 'linking %d editions' % len(chunk)
        
        for e in chunk:
            e['scan_records'] = [{'key': '/scan_record' + e['key']}]
            
        ol.save_many(chunk, 'link scan records')

# 需要导入模块: from openlibrary.api import OpenLibrary [as 别名]
# 或者: from openlibrary.api.OpenLibrary import query [as 别名]
    except:
        print('editions:', [e['key'] for e in editions])
        raise
    if all(work0 == e['works'][0] for e in editions[1:]):
        continue
    wkeys = []
    for e in editions:
        for wkey in e['works']:
            if wkey not in wkeys:
                wkeys.append(wkey)

    works = []
    for wkey in wkeys:
        w = ol.get(wkey)
        q = {'type': '/type/edition', 'works': wkey, 'limit': 1000}
        w['number_of_editions'] = len(ol.query(q))
        works.append(w)
    title0 = works[0]['title'].lower()
    if not all(w['title'].lower() == title0 for w in works[1:]):
        continue
    print(ia, ekeys)
    print('  works:', wkeys)
    def work_key_int(wkey):
        return int(re_work_key.match(wkey).group(1))
    works = sorted(works, cmp=lambda a,b:-cmp(a['number_of_editions'],b['number_of_editions']) or cmp(work_key_int(a['key']), work_key_int(b['key'])))
    print('  titles:', [(w['title'], w['number_of_editions']) for w in works])
    print(author0)
    #print [w['authors'][0]['author'] for w in works]
    assert all(author0 == w['authors'][0]['author'] for w in works)
    merge_works(works)
    print()

# 需要导入模块: from openlibrary.api import OpenLibrary [as 别名]
# 或者: from openlibrary.api.OpenLibrary import query [as 别名]
                'authors': [{'key': akey}],
                'works': [{'key': wkey}],
                'ocaid': e['ia'],
                'publishers': ['Smashwords'],
            }
            if 'isbn' in e:
                edition['isbn'] = e['isbn']
            ekey = ol.new(edition)
            print ekey, e['title']
        continue

update = {}
for k, v in authors.items():
    if not v['editions']:
        continue
    authors = ol.query({'type': '/type/author', 'name': k})
    assert authors
    if not authors:
        assert k in authors_done
        continue
        author_done.append(k)
        print {'name': k, 'bio': v['bio'] }
        for e in v['editions']:
            pprint({
                'title': e['title'],
                'authors': {'author': {'key': 'xxx'}},
                'description': e['description'],
            })
        print
        continue
        print 'bio:', `v['bio']`

# 需要导入模块: from openlibrary.api import OpenLibrary [as 别名]
# 或者: from openlibrary.api.OpenLibrary import query [as 别名]
                " and (collection like 'inlibrary%%' or collection like 'lendinglibrary%%' or (scanner is not null and scancenter is not null and scandate is null))" + \
                " and updated > %s" + \
                " order by updated", [loaded_start])
        t_start = time()

        for ia, contributor, updated, noindex, collection, ia_format, boxid in cur.fetchall():
            print updated, ia
            if contributor == 'Allen County Public Library Genealogy Center':
                print 'skipping Allen County Public Library Genealogy Center'
                continue
            collections = set()
            if collection:
                collections = set(i.lower().strip() for i in collection.split(';'))

            q = {'type': '/type/edition', 'ocaid': ia}
            if ol.query(q):
                continue
            q = {'type': '/type/edition', 'ia_loaded_id': ia}
            if ol.query(q):
                continue
            load_book(ia, collections, boxid, scanned=False)
            print >> open('loaded_start', 'w'), updated
        cur.close()

        scanned_start = open('scanned_start').readline()[:-1]
        print scanned_start
        cur = conn.cursor()
        cur.execute("select " + sql_fields + \
            " from metadata" + \
            " where mediatype='texts'" + \
                " and (not curatestate='dark' or curatestate is null)" + \

# 需要导入模块: from openlibrary.api import OpenLibrary [as 别名]
# 或者: from openlibrary.api.OpenLibrary import query [as 别名]
skip = None
add_src_rec = open('add_source_record', 'w')
new_book = open('new_book', 'w')
for line in open('/2/edward/20century/records_to_load'):
    num += 1
    full_rec = eval(line)
    item = full_rec['ia_id']
    if skip:
        if skip == item:
            skip = None
        else:
            continue
    if show_progress:
        print '%d/%d %d %.2f%% %s' % (num, total, load_count, (float(num) * 100.0) / total, item)
    if check_for_existing:
        if ol.query({'type': '/type/edition', 'ocaid': item}):
            print item, 'already loaded'
            load_count += 1
            continue
        if ol.query({'type': '/type/edition', 'source_records': 'ia:' + ia}):
            print 'already loaded'
            load_count += 1
            continue
    try:
        assert not re_census.match(item)
        assert 'passportapplicat' not in item
        assert len(full_rec.keys()) != 1
    except AssertionError:
        print item
        raise
    filename = '/2/edward/20century/scans/' + item[:2] + '/' + item + '/' + item + '_marc.xml'

# 需要导入模块: from openlibrary.api import OpenLibrary [as 别名]
# 或者: from openlibrary.api.OpenLibrary import query [as 别名]
ol = OpenLibrary('http://openlibrary.org/')

local_db = MySQLdb.connect(db='merge_editions')
local_cur = conn.cursor()

archive_db = MySQLdb.connect(host=ia_db_host, user=ia_db_user, \
        passwd=ia_db_pass, db='archive')
archive_cur = conn.cursor()

fields = ['identifier', 'updated', 'collection']
sql_fields = ', '.join(fields)

archive_cur.execute("select " + sql_fields + \
    " from metadata" + \
    " where scanner is not null and mediatype='texts'" + \
        " and (not curatestate='dark' or curatestate is null)" + \
        " and collection is not null and boxid is not null and identifier not like 'zdanh_test%' and scandate is not null " + \
        " order by updated")

for num, (ia, updated, collection) in enumerate(cur.fetchall()):
    if 'lending' not in collection and 'inlibrary' not in collection:
        continue
    q = {'type': '/type/edition', 'ocaid': ia}
    editions = set(str(i) for i in ol.query(q))
    q = {'type': '/type/edition', 'source_records': 'ia:' + ia}
    editions.update(str(i) for i in ol.query(q))
    if len(editions) > 1:
        print((ia, list(editions)))
        local_cur.execute('replace into merge (ia, editions) values (%s, %s)', [ia, ' '.join(editions)])

# 需要导入模块: from openlibrary.api import OpenLibrary [as 别名]
# 或者: from openlibrary.api.OpenLibrary import query [as 别名]
#            'description': edition['description'],
#            'subjects': ['Lending library'],
#        })
#
        wkey = edition['work']
        w = ol.get(wkey)
        assert edition['description']
        if not w.get('description'):
            w['description'] = edition['description']
        if 'Lending library' not in w.get('subjects', []):
            w.setdefault('subjects', []).append('Lending library')
            ol.save(wkey, w, 'Add lending edition from Smashwords')
        continue

        q = {'type': '/type/edition', 'ocaid': edition['ia']}
        existing = list(ol.query(q))
        if existing:
            print existing
            print 'skip existing:', str(existing[0]), edition['ia']
            continue

        e = {
            'type': '/type/edition',
            'title': edition['title'],
            'authors': [{'key': akey}],
            'works': [{'key': wkey}],
            'ocaid': edition['ia'],
            'publishers': ['Smashwords'],
            'publish_date': edition['publish_date'],
        }
        if 'isbn' in edition:

# 需要导入模块: from openlibrary.api import OpenLibrary [as 别名]
# 或者: from openlibrary.api.OpenLibrary import query [as 别名]
class VacuumBot:
  """VacuumBot can help clean up Open Library, just tell it what to do!
  
  The VacuumBot has (its) methods to do specific cleanup tasks.
  It needs the credentials of a bot account on Open Library and some instructions.
  Naturally, it shows no mercy.
  """
  
  def __init__(self, username, password):
    """Takes a username and password of a bot account to establish a connection to OL.
    
    """
    self.ol = OpenLibrary()
    self.ol.login(username, password)
    self.pagreg = re.compile(r"[^\s]\s+[:;]$")
    self.emptypagreg = re.compile(r"[,.:;]+$")
    self.formatdict = simplejson.load(codecs.open("formatdict.json", "rb", "utf-8"))
    self.enc2 = codecs.getencoder("ascii")
    self.savebuffer = {}
    self.badrecords = []
    self.aucache = {}
    self.wocache = {}
    #self.formatcache = NKCache("ol_books_formats", api_key = "cfdeaeda-4a22-4ae7-a2bf-1634da98fa1b")
    self.logfile = codecs.EncodedFile(open("vacuumbot-log.tsv", "ab"), "unicode_internal", "utf-8", "replace")
  
  def enc(self, str):
    return self.enc2(str, "backslashreplace")[0]
  
  def flog(self, key, operation, message):
    """Log to file 'vacuumbot-log.tsv'. Lines are time, key, operation and message, tab-separated.
    """
    self.logfile.write(unicode(strftime("%Y-%m-%d_%H:%M:%S", localtime()) + "\t" + key + "\t" + operation + "\t" + message + "\n"))
  
  def save_error(self, key, message):
    errorfile = codecs.EncodedFile(open("vacuumbot-errors.txt", "ab"), "unicode_internal", "utf-8", "replace")
    errorfile.write(unicode("[" + strftime("%Y-%m-%d_%H:%M:%S", localtime()) + "] Could not save record for: " + key + ", error was: " + message + "\n"))
    errorfile.close()
  
  def query(self, query):
    return self.ol.query(query)
  
  def ol_save(self, key, record, message):
    try:
      self.ol.save(key, record, self.enc(message))
      self.flog(key, "direct save", message)
      print_log("Saved "+key+": "+message)
    except OLError as e:
      self.save_error(key, str(e))
      print_log("Save failed: "+str(e))
  
  def ol_save2(self, key, record, message):
    if message != None:
      record = marshal(record)
      if message in self.savebuffer.keys():
        self.savebuffer[message][key] = record
        if len(self.savebuffer[message]) >= 100:
          self.flush(message)
      else:
        self.savebuffer[message] = {}
        self.savebuffer[message][key] = record
      self.flog(key, "buffer save", message)
    else:
      raise Exception("Message for saving is missing!")
  
  def flush(self, buffer_name):
    try:
      if len(self.savebuffer[buffer_name]) > 0:
        self.ol.save_many(self.savebuffer[buffer_name].values(), self.enc(buffer_name))
        for key in self.savebuffer[buffer_name].keys():
          self.flog(key, "buffer flush", buffer_name)
        print_log("Flushed buffer ("+str(len(self.savebuffer[buffer_name]))+" records): "+buffer_name)
        self.savebuffer[buffer_name] = {}
        sleep(1)
    except OLError as e:
      # Try to remove rejected record from buffer
      err_mess = simplejson.loads(re.sub(r'^[^{]*', "", str(e)))
      if err_mess["error"] == "bad_data":
        k = err_mess["at"]["key"]
        del self.savebuffer[buffer_name][k]
        self.save_error(k, "Multisave failed: "+str(e)+"; removed record from buffer")
      else:
        k = self.savebuffer[buffer_name].keys()[0]
        self.save_error(k, "Multisave failed: "+str(e))
  
  def flush_all(self):
    for m in self.savebuffer.keys():
      self.flush(m)
  
  def ol_get(self, key, v=None):
    """Gets a record from OL and catches OLErrors.
    
    Make sure you check for None when you process this function's result.
    """
    try:
      return self.ol.get(key, v)
    except OLError as e:
      self.save_error(key, str(e))
      print_log("Get failed: "+str(e))
  
  def clean_author_dates(self):
#.........这里部分代码省略.........