Python olapi.OpenLibrary類代碼示例

from catalog.utils.query import query_iter, set_staging, withKey, get_mc
import sys, codecs, re
sys.path.append('/home/edward/src/olapi')
from olapi import OpenLibrary, Reference
from catalog.read_rc import read_rc
from catalog.get_ia import get_from_archive, get_from_local
from catalog.marc.fast_parse import get_first_tag, get_all_subfields
rc = read_rc()

sys.stdout = codecs.getwriter('utf-8')(sys.stdout)
set_staging(True)

ol = OpenLibrary("http://dev.openlibrary.org")
ol.login('EdwardBot', rc['EdwardBot'])

q = { 'type': '/type/edition', 'table_of_contents': None, 'subjects': None }
queue = []
count = 0
for e in query_iter(q, limit=100):
    key = e['key']
    mc = get_mc(key)
    if not mc:
        continue
    data = get_from_local(mc)
    line = get_first_tag(data, set(['041']))
    if not line:
        continue
    print key, line[0:2], list(get_all_subfields(line))

import os, re, sys, codecs
from openlibrary.catalog.read_rc import read_rc
from openlibrary.catalog.importer.db_read import get_mc

sys.path.append('/home/edward/src/olapi')
from olapi import OpenLibrary, unmarshal, marshal

rc = read_rc()
ol = OpenLibrary("http://dev.openlibrary.org")
ol.login('EdwardBot', rc['EdwardBot']) 

test_dir = '/home/edward/ol/test_data'

re_edition = re.compile('^/b/OL\d+M$')

re_meta_mrc = re.compile('^([^/]*)_meta.mrc:0:\d+$')

#out = open('source_records', 'w')
for f in os.listdir(test_dir):
    key = f.replace('_', '/')
    if not re_edition.match(key):
        continue
    print key
    continue
    mc = get_mc(key)
    print key, mc
    if not mc:
        continue
    e = ol.get(key)
    if e.get('source_records', []):
        continue

#!/usr/bin/env python
# NondescriptBot
# by John Shutt (http://shutt.in)

import sys
from olapi import OpenLibrary
# secrets.py holds the login info, and is excluded from version control
from secrets import login_name, password

ol = OpenLibrary()

# Log in.
logged_in = False
print 'Trying to log in...'
for attempt in range(5):
    try:
        ol.login(login_name, password)
        logged_in = True
        print 'Login successful.'
        break
    except:
        print 'ol.login() error; retrying'
if not logged_in:
    sys.exit('Failed to log in.')

#!/usr/bin/env python

from time import localtime, sleep, strftime
from olapi import OpenLibrary

ol = OpenLibrary()
ol.login("someBot", "somePassword")

def print_log(msg):
  timestamp = strftime("%Y%m%d_%H:%M:%S", localtime())
  print("[" + timestamp + "] " + msg)

def set_identifier(book, id_name, id_value):
  ids = book.setdefault("identifiers", {})
  ids[id_name] = [id_value]

def set_goodreads_id(olid, goodreads_id):
  book = ol.get(olid)
  set_identifier(book, "goodreads", goodreads_id)
  ol.save(book['key'], book, "Added goodreads ID.")

def map_id(olid, isbn, goodreads_id):
  book = ol.get(olid)
  if book.has_key('identifiers'):
    if book['identifiers'].has_key('goodreads'):
      if goodreads_id in book['identifiers']['goodreads']:
        return
  print_log("Adding Goodreads ID \"" + goodreads_id + "\" to Openlibrary ID \"" + olid + "\"")
  set_goodreads_id(olid, goodreads_id)

def load(filename):

from catalog.utils.query import query_iter, set_staging, withKey
import sys, codecs, re

sys.path.append("/home/edward/src/olapi")
from olapi import OpenLibrary, Reference
from catalog.read_rc import read_rc

rc = read_rc()

sys.stdout = codecs.getwriter("utf-8")(sys.stdout)
set_staging(True)

ol = OpenLibrary("http://dev.openlibrary.org")
ol.login("EdwardBot", rc["EdwardBot"])

re_skip = re.compile("\b([A-Z]|Co|Dr|Jr|Capt|Mr|Mrs|Ms|Prof|Rev|Revd|Hon)\.$")


def has_dot(s):
    return s.endswith(".") and not re_skip.search(s)


q = {"type": "/type/edition", "table_of_contents": None, "subjects": None}
queue = []
count = 0
for e in query_iter(q):
    if not e.get("subjects", None) or not any(has_dot(s) for s in e["subjects"]):
        continue
    subjects = [s[:-1] if has_dot(s) else s for s in e["subjects"]]
    q = {"key": e["key"], "subjects": {"connect": "update_list", "value": subjects}}
    if e.get("table_of_contents", None) and e["table_of_contents"][0]["type"] == "/type/text":

import sys
import codecs
import re
sys.path.append('/home/edward/src/olapi')
from olapi import OpenLibrary, Reference
from catalog.read_rc import read_rc

import six


rc = read_rc()

sys.stdout = codecs.getwriter('utf-8')(sys.stdout)
set_staging(True)

ol = OpenLibrary("http://dev.openlibrary.org")
ol.login('EdwardBot', rc['EdwardBot'])

re_skip = re.compile('\b([A-Z]|Co|Dr|Jr|Capt|Mr|Mrs|Ms|Prof|Rev|Revd|Hon)\.$')

def has_dot(s):
    return s.endswith('.') and not re_skip.search(s)

q = { 'type': '/type/edition', 'table_of_contents': None, 'subjects': None }
queue = []
count = 0
for e in query_iter(q):
    if not e.get('subjects', None) or not any(has_dot(s) for s in e['subjects']):
        continue
    subjects = [s[:-1] if has_dot(s) else s for s in e['subjects']]
    q = {

 def __init__(self, username, password):
   self.ol = OpenLibrary()
   self.ol.login(username, password)

#!/usr/bin/env python

from olapi import OpenLibrary

ol = OpenLibrary()
ol.login("VacuumBot", "somePassword")

# Top level classifications don't go in the classifications dict.
tl_classifications = ["lc_classifications","dewey_decimal_class"]

def upgrade_classifications(olid):
  """Changes classification from list of (name,value)-dict
  to dict of lists.
  """
  record = ol.get(olid)
  # Check if the classifications are a list:
  if not isinstance(record["classifications"], list):
    return
  
  # Create a new dict to replace the list:
  c = {}
  
  # Read the dicts from the classifications list:
  for k in record["classifications"]:
    if k["name"] in tl_classifications:
      if k["name"] in record.keys():
        record[k["name"]].append(k["value"])
      else:
        record[k["name"]] = [k["value"]]
    elif k["name"] not in c.keys():
      c["name"] = [k["value"]]

from openlibrary.catalog.merge.merge_marc import *
from openlibrary.catalog.read_rc import read_rc
import openlibrary.catalog.merge.amazon as amazon
from openlibrary.catalog.get_ia import *
from openlibrary.catalog.importer.db_read import withKey, get_mc
import openlibrary.catalog.marc.fast_parse as fast_parse
import xml.parsers.expat
import web, sys

sys.path.append("/home/edward/src/olapi")
from olapi import OpenLibrary
from time import sleep

rc = read_rc()

ol = OpenLibrary("http://openlibrary.org")
ol.login("ImportBot", rc["ImportBot"])

ia_db = web.database(dbn="mysql", db="archive", user=rc["ia_db_user"], pw=rc["ia_db_pass"], host=rc["ia_db_host"])
ia_db.printing = False

re_meta_marc = re.compile("([^/]+)_(meta|marc)\.(mrc|xml)")

threshold = 875
amazon.set_isbn_match(225)


def try_amazon(thing):
    if "isbn_10" not in thing:
        return None
    if "authors" in thing:

from openlibrary.catalog.merge.merge_marc import build_marc
from openlibrary.catalog.importer.db_read import get_mc, withKey
sys.path.append('/home/edward/src/olapi')
from olapi import OpenLibrary, unmarshal

from openlibrary.catalog.read_rc import read_rc

rc = read_rc()

marc_index = web.database(dbn='postgres', db='marc_index')
marc_index.printing = False

db_amazon = web.database(dbn='postgres', db='amazon')
db_amazon.printing = False

ol = OpenLibrary("http://openlibrary.org")
ol.login('ImportBot', rc['ImportBot']) 

sys.stdout = codecs.getwriter('utf-8')(sys.stdout)

t0 = time()
t_prev = time()
rec_no = 0
chunk = 50
load_count = 0

archive_id = sys.argv[1]

def get_with_retry(key):
    for i in range(3):
        try: