Python OpenLibrary.save_many方法代码示例

# 需要导入模块: from openlibrary.api import OpenLibrary [as 别名]
# 或者: from openlibrary.api.OpenLibrary import save_many [as 别名]
def main(server):
    ol = OpenLibrary(server)
    ol.autologin()

    volumes = ol.query({'type': '/type/volume', 'limit': False, '*': None, 'edition': {'*': None}})

    volumes = dict((v['key'], v) for v in volumes)
    editions = dict((v['edition']['key'], v['edition']) for v in volumes.values() if v['edition'])

    def make_volume(v):
        d = {}
        v.pop('edition')
        v['type'] = {'key': '/type/volume'}
        for k in ['type', 'ia_id', 'volume_number']:
            if k in v:
                d[k] = v[k]
        return d

    for e in editions.values():
        e['volumes'] = []

    for v in volumes.values():
        if v.get('edition'):
            e = editions[v.get('edition')['key']]
            e['volumes'].append(make_volume(v))

    for e in editions.values():
        e['volumes'] = sorted(e['volumes'], key=lambda v: v['volume_number'])

    print 'linking volumes to %d editions' % len(editions)
    ol.save_many(editions.values(), 'link volumes')

# 需要导入模块: from openlibrary.api import OpenLibrary [as 别名]
# 或者: from openlibrary.api.OpenLibrary import save_many [as 别名]
def load():
    """Loads documents to http://0.0.0.0:8080"""
    documents = {}
    for f in find("data"):
        doc = simplejson.load(open(f))
        documents[doc['key']] = doc

    keys = topological_sort(documents.keys(), 
            get_children=lambda key: [k for k in get_references(documents[key]) if not k.startswith("/type/")])

    from openlibrary.api import OpenLibrary
    ol = OpenLibrary("http://0.0.0.0:8080")
    ol.autologin()
    print ol.save_many([documents[k] for k in keys], comment="documents copied from openlibrary.org")

# 需要导入模块: from openlibrary.api import OpenLibrary [as 别名]
# 或者: from openlibrary.api.OpenLibrary import save_many [as 别名]
def main():
    ol = OpenLibrary()
    ol.autologin()

    plugin = sys.argv[1]

    all_docs = []

    for pattern in sys.argv[2:]:
        docs = ol.query({"key~": pattern, "*": None}, limit=1000)
        all_docs.extend(docs)

    for doc in all_docs:
        doc['plugin'] = plugin

    print ol.save_many(all_docs, comment="Marked as part of %s plugin." % plugin)

# 需要导入模块: from openlibrary.api import OpenLibrary [as 别名]
# 或者: from openlibrary.api.OpenLibrary import save_many [as 别名]
def main(server):
    ol = OpenLibrary(server)
    ol.autologin()

    scan_records = ol.query(type='/type/scan_record', limit=False, edition={'*': None})
    editions = (r['edition'] for r in scan_records)

    # process 1000 editions at a time.
    while True:
        chunk = take(1000, editions)
        if not chunk:
            break

        print 'linking %d editions' % len(chunk)
        
        for e in chunk:
            e['scan_records'] = [{'key': '/scan_record' + e['key']}]
            
        ol.save_many(chunk, 'link scan records')

# 需要导入模块: from openlibrary.api import OpenLibrary [as 别名]
# 或者: from openlibrary.api.OpenLibrary import save_many [as 别名]
            else:
                book['identifiers']['librarything'] = [ltid]
        else:
            book['identifiers'] = {'librarything': [ltid]}
        if newbook:
            newlist.append(book)

    # Save the data back to the site
    # If there's nothing to update, skip save_many
    if len(newlist) == 0:
         continue
    saved = False
    #for attempt in range(5):
    try:
         print 'Trying to save_many'
         print ol.save_many(newlist, 'added LibraryThing ID')
         saved = True
    except KeyboardInterrupt:
         sys.exit(0)
    except:
         badlist = []
         for e in newlist:
              for akey in e.get('authors', []):
                   got_author = False
                   for attempt in range(5):
                        try:
                             a = ol.get(akey['key'])
                             got_author = True
                             break
                        except:
                             print 'ol.get(author) error; retrying.'

# 需要导入模块: from openlibrary.api import OpenLibrary [as 别名]
# 或者: from openlibrary.api.OpenLibrary import save_many [as 别名]
        if k == 'authors':
            min_author = set(min((e.get('authors', []) for e in editions), key=len))
            if all(min_author <= set(e.get('authors', [])) for e in editions):
                master[k] = max((e.get('authors', []) for e in editions), key=len)
                continue

        print('unmerged field:', k)
        print([e.get(k) for e in editions])
        no_merge = True
    if no_merge:
        continue
    if 'location' in master and isinstance(master['location'], six.string_types) and master['location'].startswith('/books/'):
        del master['location']
    updates.append(master)
    for e in editions[1:]:
        redirect = {
            'type': Reference('/type/redirect'),
            'location': min_ekey,
            'key': e['key'],
        }
        updates.append(redirect)
    print(len(updates), min_ekey)
    try:
        print(ol.save_many(updates, 'merge lending editions'))
    except:
        for i in updates:
            print(i)
        raise
    cur2.execute('update merge set done=now() where ia=%s', [ia])

# 需要导入模块: from openlibrary.api import OpenLibrary [as 别名]
# 或者: from openlibrary.api.OpenLibrary import save_many [as 别名]
for w in query_iter(work_q):
    if not w.get('authors'):
        print('no authors')
        continue
    if any(isinstance(a, dict) and 'author' in a for a in w['authors']):
        continue
    print(len(queue), w['key'], w['title']) # , ol.get(w['authors'][0]['key'])['name']
    full = ol.get(w['key'])
    authors = full['authors']
    assert all(isinstance(a, Reference) for a in authors)
    full['authors'] = [{'author':a} for a in authors]
    queue.append(full)
    if len(queue) > 1000:
        print('saving')
        print(ol.save_many(queue, 'update format of authors in works to provide roles'))
        queue = []
        print('two second pause')
        sleep(2)
    continue
    work_e = {
        'type': '/type/edition',
        'works': w['key'],
        'by_statement': None,
    }
    for e in query_iter(work_e):
        by = e['by_statement']
        if by:
            print('  ', e['key'], by)

print('saving')

# 需要导入模块: from openlibrary.api import OpenLibrary [as 别名]
# 或者: from openlibrary.api.OpenLibrary import save_many [as 别名]
        for a in existing_work.get('authors', []):
            obj = withKey(a['author'])
            if obj['type']['key'] != '/type/redirect':
                continue
            new_akey = obj['location']
            a['author'] = {'key': new_akey}
            assert new_akey.startswith('/authors/')
            obj = withKey(new_akey)
            assert obj['type']['key'] == '/type/author'
        print >> fh_log, 'existing:', existing_work
        print >> fh_log, 'subtitle:', `existing_work['subtitle']` if 'subtitle' in existing_work else 'n/a'
        update.append(existing_work)
        work_updated.append(best)
    if do_updates:
        try:
            print >> fh_log, ol.save_many(update, 'merge works')
        except:
            for page in update:
                print page
            raise
    return work_updated

def update_works(akey, works, do_updates=False):
    # we can now look up all works by an author   
    if do_updates:
        rc = read_rc()
        ol.login('WorkBot', rc['WorkBot']) 
    assert do_updates

    fh_log = open('/1/var/log/openlibrary/work_finder/' + strftime('%F_%T'), 'w')
    works = list(works)

# 需要导入模块: from openlibrary.api import OpenLibrary [as 别名]
# 或者: from openlibrary.api.OpenLibrary import save_many [as 别名]
            w = withKey(k)
            if w['type']['key'] == '/type/redirect':
                print 'redirect found'
                wkey = w['location']
                assert re_work_key.match(wkey)
                for ekey in editions:
                    e = withKey(ekey)
                    e['works'] = [Reference(wkey)]
                    fix_redirects.append(e)
                continue
            work_title[k] = w['title']
        if not fix_redirects:
            print 'no redirects left'
            break
        print 'save redirects'
        ol.save_many(fix_redirects, "merge works")

    all_existing = set()
    work_keys = []
    for w in works:
        existing = set()
        for e in w['editions']:
            existing.update(edition_to_work[e])
        if not existing: # editions have no existing work
            if do_updates:
                wkey = ol.new({'title': w['title'], 'type': '/type/work'})
            print 'new work:', wkey, `w['title']`
            #print 'new work:', `w['title']`
            update = []
            for ekey in w['editions']:
                e = ol.get(ekey)

# 需要导入模块: from openlibrary.api import OpenLibrary [as 别名]
# 或者: from openlibrary.api.OpenLibrary import save_many [as 别名]
class VacuumBot:
  """VacuumBot can help clean up Open Library, just tell it what to do!
  
  The VacuumBot has (its) methods to do specific cleanup tasks.
  It needs the credentials of a bot account on Open Library and some instructions.
  Naturally, it shows no mercy.
  """
  
  def __init__(self, username, password):
    """Takes a username and password of a bot account to establish a connection to OL.
    
    """
    self.ol = OpenLibrary()
    self.ol.login(username, password)
    self.pagreg = re.compile(r"[^\s]\s+[:;]$")
    self.emptypagreg = re.compile(r"[,.:;]+$")
    self.formatdict = simplejson.load(codecs.open("formatdict.json", "rb", "utf-8"))
    self.enc2 = codecs.getencoder("ascii")
    self.savebuffer = {}
    self.badrecords = []
    self.aucache = {}
    self.wocache = {}
    #self.formatcache = NKCache("ol_books_formats", api_key = "cfdeaeda-4a22-4ae7-a2bf-1634da98fa1b")
    self.logfile = codecs.EncodedFile(open("vacuumbot-log.tsv", "ab"), "unicode_internal", "utf-8", "replace")
  
  def enc(self, str):
    return self.enc2(str, "backslashreplace")[0]
  
  def flog(self, key, operation, message):
    """Log to file 'vacuumbot-log.tsv'. Lines are time, key, operation and message, tab-separated.
    """
    self.logfile.write(unicode(strftime("%Y-%m-%d_%H:%M:%S", localtime()) + "\t" + key + "\t" + operation + "\t" + message + "\n"))
  
  def save_error(self, key, message):
    errorfile = codecs.EncodedFile(open("vacuumbot-errors.txt", "ab"), "unicode_internal", "utf-8", "replace")
    errorfile.write(unicode("[" + strftime("%Y-%m-%d_%H:%M:%S", localtime()) + "] Could not save record for: " + key + ", error was: " + message + "\n"))
    errorfile.close()
  
  def query(self, query):
    return self.ol.query(query)
  
  def ol_save(self, key, record, message):
    try:
      self.ol.save(key, record, self.enc(message))
      self.flog(key, "direct save", message)
      print_log("Saved "+key+": "+message)
    except OLError as e:
      self.save_error(key, str(e))
      print_log("Save failed: "+str(e))
  
  def ol_save2(self, key, record, message):
    if message != None:
      record = marshal(record)
      if message in self.savebuffer.keys():
        self.savebuffer[message][key] = record
        if len(self.savebuffer[message]) >= 100:
          self.flush(message)
      else:
        self.savebuffer[message] = {}
        self.savebuffer[message][key] = record
      self.flog(key, "buffer save", message)
    else:
      raise Exception("Message for saving is missing!")
  
  def flush(self, buffer_name):
    try:
      if len(self.savebuffer[buffer_name]) > 0:
        self.ol.save_many(self.savebuffer[buffer_name].values(), self.enc(buffer_name))
        for key in self.savebuffer[buffer_name].keys():
          self.flog(key, "buffer flush", buffer_name)
        print_log("Flushed buffer ("+str(len(self.savebuffer[buffer_name]))+" records): "+buffer_name)
        self.savebuffer[buffer_name] = {}
        sleep(1)
    except OLError as e:
      # Try to remove rejected record from buffer
      err_mess = simplejson.loads(re.sub(r'^[^{]*', "", str(e)))
      if err_mess["error"] == "bad_data":
        k = err_mess["at"]["key"]
        del self.savebuffer[buffer_name][k]
        self.save_error(k, "Multisave failed: "+str(e)+"; removed record from buffer")
      else:
        k = self.savebuffer[buffer_name].keys()[0]
        self.save_error(k, "Multisave failed: "+str(e))
  
  def flush_all(self):
    for m in self.savebuffer.keys():
      self.flush(m)
  
  def ol_get(self, key, v=None):
    """Gets a record from OL and catches OLErrors.
    
    Make sure you check for None when you process this function's result.
    """
    try:
      return self.ol.get(key, v)
    except OLError as e:
      self.save_error(key, str(e))
      print_log("Get failed: "+str(e))
  
  def clean_author_dates(self):
#.........这里部分代码省略.........