Python OpenLibrary.get方法代码示例

# 需要导入模块: from openlibrary.api import OpenLibrary [as 别名]
# 或者: from openlibrary.api.OpenLibrary import get [as 别名]
skip = 'elephantatwaldor00mira'
skip = 'sybasesqlserverp00paul'
skip = 'karmadunl00dunl'
skip = 'norbychronicles00asim'
skip = 'elizabethbarrett00fors'
skip = None
updates = []
cur.execute('select ia, editions, done, unmerge_count from merge')
for ia, ekeys, done, unmerge_count in cur.fetchall():
    if skip:
        if ia == skip:
            skip = None
        else:
            continue
    ekeys = ['/books/OL%dM' % x for x in sorted(int(re_edition_key.match(ekey).group(1)) for ekey in ekeys.split(' '))]
    editions = [ol.get(ekey) for ekey in ekeys]

    if any('authors' not in e or 'works' not in e for e in editions):
        continue
    author0 = editions[0]['authors'][0]
    work0 = editions[0]['works'][0]
    try:
        if not all(author0 == e['authors'][0] for e in editions[1:]):
            continue
    except:
        print('editions:', [e['key'] for e in editions])
        raise
    if all(work0 == e['works'][0] for e in editions[1:]):
        continue
    wkeys = []
    for e in editions:

# 需要导入模块: from openlibrary.api import OpenLibrary [as 别名]
# 或者: from openlibrary.api.OpenLibrary import get [as 别名]
re_nonword = re.compile(r'\W', re.U)

conn = MySQLdb.connect(db='merge_editions')
cur = conn.cursor()
cur2 = conn.cursor()

ol = OpenLibrary('http://openlibrary.org/')
ol.login('EdwardBot', 'As1Wae9b')

cur.execute('select ia, editions, done from merge where done is null and unmerge_count=0')
for ia, ekeys, done in cur.fetchall():
    updates = []
    ekeys = ['/books/OL%dM' % x for x in sorted(int(re_edition_key.match(ekey).group(1)) for ekey in ekeys.split(' '))]
    print((ia, ekeys))
    min_ekey = ekeys[0]
    editions = [ol.get(ekey) for ekey in ekeys]
    master = editions[0]

    for e in editions:
        for k in 'classifications', 'identifiers', 'table_of_contents':
            if k in e and not e[k]:
                del e[k]

    all_keys = set()
    for e in editions:
        all_keys.update(k for k, v in e.items() if v)
    for k in 'latest_revision', 'revision', 'created', 'last_modified', 'key', 'type', 'genres':
        if k in all_keys:
            all_keys.remove(k)

    for k in all_keys.copy():

# 需要导入模块: from openlibrary.api import OpenLibrary [as 别名]
# 或者: from openlibrary.api.OpenLibrary import get [as 别名]
 saved = False
 #for attempt in range(5):
 try:
      print 'Trying to save_many'
      print ol.save_many(newlist, 'added LibraryThing ID')
      saved = True
 except KeyboardInterrupt:
      sys.exit(0)
 except:
      badlist = []
      for e in newlist:
           for akey in e.get('authors', []):
                got_author = False
                for attempt in range(5):
                     try:
                          a = ol.get(akey['key'])
                          got_author = True
                          break
                     except:
                          print 'ol.get(author) error; retrying.'
                if not got_author:
                     sys.exit('Failed to get author: %r' % akey['key'])
                if a['type'] == '/type/author':
                     continue
                if badlist.count(e) == 0:
                     badlist.append(e)
      for badbook in badlist:
           newlist.remove(badbook)
           f.write(badbook['key'])
           f.write('\n')
      print ol.save_many(newlist, 'added LibraryThing ID')

# 需要导入模块: from openlibrary.api import OpenLibrary [as 别名]
# 或者: from openlibrary.api.OpenLibrary import get [as 别名]
work_q = {
    'type': '/type/work',
    'authors': None,
    'title': None,
}

queue = []

for w in query_iter(work_q):
    if not w.get('authors'):
        print('no authors')
        continue
    if any(isinstance(a, dict) and 'author' in a for a in w['authors']):
        continue
    print(len(queue), w['key'], w['title']) # , ol.get(w['authors'][0]['key'])['name']
    full = ol.get(w['key'])
    authors = full['authors']
    assert all(isinstance(a, Reference) for a in authors)
    full['authors'] = [{'author':a} for a in authors]
    queue.append(full)
    if len(queue) > 1000:
        print('saving')
        print(ol.save_many(queue, 'update format of authors in works to provide roles'))
        queue = []
        print('two second pause')
        sleep(2)
    continue
    work_e = {
        'type': '/type/edition',
        'works': w['key'],
        'by_statement': None,

# 需要导入模块: from openlibrary.api import OpenLibrary [as 别名]
# 或者: from openlibrary.api.OpenLibrary import get [as 别名]
rec_no = 0
chunk = 50
load_count = 0

archive_id = sys.argv[1]

def get_with_retry(key):
    for i in range(3):
        try:
            return ol.get(key)
        except urllib2.HTTPError, error:
            if error.code != 500:
                raise
        print 'retry save'
        sleep(10)
    return ol.get(key)

def save_with_retry(key, data, comment):
    for i in range(3):
        try:
            return ol.save(key, data, comment)
        except urllib2.HTTPError, error:
            if error.code != 500:
                raise
        print 'retry save'
        sleep(10)

# urllib2.HTTPError: HTTP Error 500: Internal Server Error

def percent(a, b):
    return float(a * 100.0) / b

# 需要导入模块: from openlibrary.api import OpenLibrary [as 别名]
# 或者: from openlibrary.api.OpenLibrary import get [as 别名]
        print 'ol.autologin() error; retrying'
conn = sqlite3.connect(db)
c = conn.cursor()
reader = csv.reader(open(csvfile), delimiter='\t', quotechar='|')
for row in reader:
    ltid = row[0]
    olid = row[1]
    key = '/books' + olid[olid.rindex('/'):len(olid)]
    c.execute('select * from ids where key = ?', (key,))
    x = c.fetchone()
    if x != None:
        continue
    print 'Trying to get key: %r' % key
    for attempt in range(5):
        try:
            data = ol.get(key)
            break
        except:
            print 'ol.get() error; retrying'
    if data.has_key('identifiers'):
        if data['identifiers'].has_key('librarything'):
            if data['identifiers']['librarything'].count(ltid) == 0:
                data['identifiers']['librarything'].append(ltid)
        else:
            data['identifiers']['librarything'] = [ltid]
    else:
        data['identifiers'] = {'librarything': [ltid]}
    for attempt in range(5):
        try:
            print ol.save(key, data, 'added LibraryThing ID')
            break

# 需要导入模块: from openlibrary.api import OpenLibrary [as 别名]
# 或者: from openlibrary.api.OpenLibrary import get [as 别名]
from openlibrary.api import OpenLibrary

ol = OpenLibrary('http://openlibrary.org/')

data = eval(open('update').read())

done = []

for author in data:
    print (author['key'], author['name'])
    akey = author['key']
    a = ol.get(akey)
    if not a.get('bio') and author['bio']:
        a['bio'] = author['bio']
        ol.save(akey, a, 'Add author bio from Smashwords.')

    for edition in author['editions']:
#        wkey = ol.new({
#            'type': '/type/work',
#            'title': edition['title'],
#            'authors': [{'author': {'key': akey}}],
#            'description': edition['description'],
#            'subjects': ['Lending library'],
#        })
#
        wkey = edition['work']
        w = ol.get(wkey)
        assert edition['description']
        if not w.get('description'):
            w['description'] = edition['description']
        if 'Lending library' not in w.get('subjects', []):

# 需要导入模块: from openlibrary.api import OpenLibrary [as 别名]
# 或者: from openlibrary.api.OpenLibrary import get [as 别名]
        ol.save_many(fix_redirects, "merge works")

    all_existing = set()
    work_keys = []
    for w in works:
        existing = set()
        for e in w['editions']:
            existing.update(edition_to_work[e])
        if not existing: # editions have no existing work
            if do_updates:
                wkey = ol.new({'title': w['title'], 'type': '/type/work'})
            print 'new work:', wkey, `w['title']`
            #print 'new work:', `w['title']`
            update = []
            for ekey in w['editions']:
                e = ol.get(ekey)
                if do_updates:
                    e['works'] = [Reference(wkey)]
                update.append(e)
            if do_updates:
                ol.save_many(update, "add editions to new work")
            work_keys.append(wkey)
        elif len(existing) == 1:
            key = list(existing)[0]
            work_keys.append(key)
            if work_title[key] == w['title']:
                print 'no change:', key, `w['title']`
            else:
                print 'new title:', key, `work_title[key]`, '->', `w['title']`
                print sorted(work_to_edition[key])
                print w['editions']

# 需要导入模块: from openlibrary.api import OpenLibrary [as 别名]
# 或者: from openlibrary.api.OpenLibrary import get [as 别名]
class VacuumBot:
  """VacuumBot can help clean up Open Library, just tell it what to do!
  
  The VacuumBot has (its) methods to do specific cleanup tasks.
  It needs the credentials of a bot account on Open Library and some instructions.
  Naturally, it shows no mercy.
  """
  
  def __init__(self, username, password):
    """Takes a username and password of a bot account to establish a connection to OL.
    
    """
    self.ol = OpenLibrary()
    self.ol.login(username, password)
    self.pagreg = re.compile(r"[^\s]\s+[:;]$")
    self.emptypagreg = re.compile(r"[,.:;]+$")
    self.formatdict = simplejson.load(codecs.open("formatdict.json", "rb", "utf-8"))
    self.enc2 = codecs.getencoder("ascii")
    self.savebuffer = {}
    self.badrecords = []
    self.aucache = {}
    self.wocache = {}
    #self.formatcache = NKCache("ol_books_formats", api_key = "cfdeaeda-4a22-4ae7-a2bf-1634da98fa1b")
    self.logfile = codecs.EncodedFile(open("vacuumbot-log.tsv", "ab"), "unicode_internal", "utf-8", "replace")
  
  def enc(self, str):
    return self.enc2(str, "backslashreplace")[0]
  
  def flog(self, key, operation, message):
    """Log to file 'vacuumbot-log.tsv'. Lines are time, key, operation and message, tab-separated.
    """
    self.logfile.write(unicode(strftime("%Y-%m-%d_%H:%M:%S", localtime()) + "\t" + key + "\t" + operation + "\t" + message + "\n"))
  
  def save_error(self, key, message):
    errorfile = codecs.EncodedFile(open("vacuumbot-errors.txt", "ab"), "unicode_internal", "utf-8", "replace")
    errorfile.write(unicode("[" + strftime("%Y-%m-%d_%H:%M:%S", localtime()) + "] Could not save record for: " + key + ", error was: " + message + "\n"))
    errorfile.close()
  
  def query(self, query):
    return self.ol.query(query)
  
  def ol_save(self, key, record, message):
    try:
      self.ol.save(key, record, self.enc(message))
      self.flog(key, "direct save", message)
      print_log("Saved "+key+": "+message)
    except OLError as e:
      self.save_error(key, str(e))
      print_log("Save failed: "+str(e))
  
  def ol_save2(self, key, record, message):
    if message != None:
      record = marshal(record)
      if message in self.savebuffer.keys():
        self.savebuffer[message][key] = record
        if len(self.savebuffer[message]) >= 100:
          self.flush(message)
      else:
        self.savebuffer[message] = {}
        self.savebuffer[message][key] = record
      self.flog(key, "buffer save", message)
    else:
      raise Exception("Message for saving is missing!")
  
  def flush(self, buffer_name):
    try:
      if len(self.savebuffer[buffer_name]) > 0:
        self.ol.save_many(self.savebuffer[buffer_name].values(), self.enc(buffer_name))
        for key in self.savebuffer[buffer_name].keys():
          self.flog(key, "buffer flush", buffer_name)
        print_log("Flushed buffer ("+str(len(self.savebuffer[buffer_name]))+" records): "+buffer_name)
        self.savebuffer[buffer_name] = {}
        sleep(1)
    except OLError as e:
      # Try to remove rejected record from buffer
      err_mess = simplejson.loads(re.sub(r'^[^{]*', "", str(e)))
      if err_mess["error"] == "bad_data":
        k = err_mess["at"]["key"]
        del self.savebuffer[buffer_name][k]
        self.save_error(k, "Multisave failed: "+str(e)+"; removed record from buffer")
      else:
        k = self.savebuffer[buffer_name].keys()[0]
        self.save_error(k, "Multisave failed: "+str(e))
  
  def flush_all(self):
    for m in self.savebuffer.keys():
      self.flush(m)
  
  def ol_get(self, key, v=None):
    """Gets a record from OL and catches OLErrors.
    
    Make sure you check for None when you process this function's result.
    """
    try:
      return self.ol.get(key, v)
    except OLError as e:
      self.save_error(key, str(e))
      print_log("Get failed: "+str(e))
  
  def clean_author_dates(self):
#.........这里部分代码省略.........