Python utils.colored_out函数代码示例

def main(ENTITY_TYPE):

    entity_type_table = ENTITY_TYPE.replace('-', '_')
    url_relationship_table = 'l_%s_url' % entity_type_table if ENTITY_TYPE != 'work' else 'l_url_%s' % entity_type_table
    main_entity_entity_point = "entity0" if ENTITY_TYPE != 'work' else "entity1"
    url_entity_point = "entity1" if ENTITY_TYPE != 'work' else "entity0"

    query = """
    WITH
        entities_wo_wikidata AS (
            SELECT DISTINCT e.id AS entity_id, e.gid AS entity_gid, u.url AS wp_url, substring(u.url from '//(([a-z]|-)+)\\.') as wp_lang
            FROM """ + entity_type_table + """ e
                JOIN """ + url_relationship_table + """ l ON l.""" + main_entity_entity_point + """ = e.id AND l.link IN (SELECT id FROM link WHERE link_type = """ + str(WIKIPEDIA_RELATIONSHIP_TYPES[ENTITY_TYPE]) + """)
                JOIN url u ON u.id = l.""" + url_entity_point + """ AND u.url LIKE 'http://%%.wikipedia.org/wiki/%%'
            WHERE 
                /* No existing WikiData relationship for this entity */
                NOT EXISTS (SELECT 1 FROM """ + url_relationship_table + """ ol WHERE ol.""" + main_entity_entity_point + """ = e.id AND ol.link IN (SELECT id FROM link WHERE link_type = """ + str(WIKIDATA_RELATIONSHIP_TYPES[ENTITY_TYPE]) + """))
                /* WP link should only be linked to this entity */
                AND NOT EXISTS (SELECT 1 FROM """ + url_relationship_table + """ ol WHERE ol.""" + url_entity_point + """ = u.id AND ol.""" + main_entity_entity_point + """ <> e.id)
                AND l.edits_pending = 0
        )
    SELECT e.id, e.gid, e.name, ewf.wp_url, b.processed
    FROM entities_wo_wikidata ewf
    JOIN """ + entity_type_table + """ e ON ewf.entity_id = e.id
    LEFT JOIN bot_wp_wikidata_links b ON e.gid = b.gid AND b.lang = ewf.wp_lang
    ORDER BY b.processed NULLS FIRST, e.id
    LIMIT 500
    """

    seen = set()
    matched = set()
    for entity in db.execute(query):
        if entity['gid'] in matched:
            continue

        colored_out(bcolors.OKBLUE, 'Looking up entity "%s" http://musicbrainz.org/%s/%s' % (entity['name'], ENTITY_TYPE, entity['gid']))
        out(' * wiki:', entity['wp_url'])

        page = WikiPage.fetch(entity['wp_url'], False)
        if page.wikidata_id:
            wikidata_url = 'http://www.wikidata.org/wiki/%s' % page.wikidata_id.upper()
            edit_note = 'From %s' % (entity['wp_url'],)
            colored_out(bcolors.OKGREEN, ' * found WikiData identifier:', wikidata_url)
            time.sleep(1)
            out(' * edit note:', edit_note.replace('\n', ' '))
            mb.add_url(ENTITY_TYPE.replace('-', '_'), entity['gid'], str(WIKIDATA_RELATIONSHIP_TYPES[ENTITY_TYPE]), wikidata_url, edit_note, True)
            matched.add(entity['gid'])

        if entity['processed'] is None and entity['gid'] not in seen:
            db.execute("INSERT INTO bot_wp_wikidata_links (gid, lang) VALUES (%s, %s)", (entity['gid'], page.lang))
        else:
            db.execute("UPDATE bot_wp_wikidata_links SET processed = now() WHERE (gid, lang) = (%s, %s)", (entity['gid'], page.lang))
        seen.add(entity['gid'])
    stats['seen'][ENTITY_TYPE] = len(seen)
    stats['matched'][ENTITY_TYPE] = len(matched)

def submit_cover_art(release, url, types):
    if already_processed(release, url):
        colored_out(bcolors.NONE, " * skipping already submitted image '%s'" % (url,))
    else:
        colored_out(bcolors.OKGREEN, " * Adding " + ",".join(types) + (" " if len(types)>0 else "") + "cover art '%s'" % (url,))
        img_file = urllib2.urlopen(url)
        im = Image.open(StringIO(img_file.read()))
        edit_note = "'''Dimension''': %sx%s\n'''Source''': %s" % (im.size[0], im.size[1], url)
        time.sleep(5)
        mb.add_cover_art(release, url, types, None, u'', edit_note, False)
        save_processed(release, url)

def submit_cover_art(release, url, types):
    if already_processed(release, url):
        colored_out(bcolors.NONE, " * skipping already submitted image '%s'" % (url,))
    else:
        colored_out(bcolors.OKGREEN, " * Adding " + ",".join(types) + (" " if len(types) > 0 else "") + "cover art '%s'" % (url,))
        content = urllib2.urlopen(url).read()
        image_file = tempfile.NamedTemporaryFile(delete=False)
        image_file.write(content)
        image_file.close()
        im = Image.open(image_file.name)
        edit_note = "'''Dimension''': %sx%s\n'''Source''': %s" % (im.size[0], im.size[1], url)
        time.sleep(5)
        mb.add_cover_art(release, image_file.name, types, None, u'', edit_note, False)
        os.remove(image_file.name)
        save_processed(release, url)

def submit_cover_art(release, url, types):
    if already_processed(release, url):
        colored_out(bcolors.NONE, " * skipping already submitted image '%s'" % (url,))
    else:
        colored_out(bcolors.OKGREEN, " * Adding " + ",".join(types) + (" " if len(types) > 0 else "") + "cover art '%s'" % (url,))
        if 'discogs' in url:
            headers = {'user-agent': 'MusicBrainzBot/0.1 +https://github.com/murdos/musicbrainz-bot'}
            resp, content = discogs_oauth_client.request(url, 'GET', headers=headers)
        else:
            content = urllib2.urlopen(url).read()
        image_file = tempfile.NamedTemporaryFile(delete=False)
        image_file.write(content)
        image_file.close()
        im = Image.open(image_file.name)
        edit_note = "'''Dimension''': %sx%s\n'''Source''': %s" % (im.size[0], im.size[1], url)
        time.sleep(5)
        mb.add_cover_art(release, image_file.name, types, None, u'', edit_note, False)
        os.remove(image_file.name)
        save_processed(release, url)

def determine_country(page):
    all_countries = set()
    all_reasons = []
    countries, reason = determine_country_from_infobox(page)
    if countries:
        all_countries.update(countries)
        all_reasons.append(reason)
    countries, reason = determine_country_from_text(page)
    if countries:
        all_countries.update(countries)
        all_reasons.append(reason)
    countries, reason, category_count = determine_country_from_categories(page)
    has_categories = False
    if countries:
        all_countries.update(countries)
        all_reasons.append(reason)
        has_categories = True
    if len(all_reasons) < 1 or not all_countries or not has_categories:
        colored_out(bcolors.WARNING, ' * not enough sources for countries', all_countries, all_reasons)
        return None, []
    if len(all_countries) > 1:
        colored_out(bcolors.FAIL, ' * conflicting countries', all_countries, all_reasons)
        return None, []
    country = list(all_countries)[0]
    colored_out(bcolors.OKGREEN, ' * new country: ', country)
    return country, all_reasons

def discogs_get_release_packaging(discogs_release):
    #if len(discogs_release.data['formats']) > 1:
    #    return None
    for format in discogs_release.data['formats']:

        if 'text' not in format:
            print 'No text found for format %s' % format['name']
            continue
        
        freetext = format['text'].lower().replace('-', '').replace(' ', '')
        colored_out(bcolors.HEADER, ' * Discogs format text: %s' % freetext)
        if 'cardboard' in freetext or 'paper' in freetext:
            return "cardboard/paper sleeve";
        elif 'digipak' in freetext or 'digipack' in freetext:
            return "digipak";
        elif 'keepcase' in freetext:
            return "keep case";
        elif 'jewel' in freetext:
            if 'slim' in freetext:
                return "slim jewel case"
            else:
                return "jewel case"

    return None

def determine_type(page):
    all_types = set()
    all_reasons = []
    types, reason = determine_type_from_page(page)
    if types:
        all_types.update(types)
        all_reasons.append(reason)
    if not all_reasons:
        colored_out(bcolors.WARNING, ' * not enough sources for types')
        return None, []
    if len(all_types) > 1:
        colored_out(bcolors.FAIL, ' * conflicting types', all_types, all_reasons)
        return None, []
    type = list(all_types)[0]
    colored_out(bcolors.OKGREEN, ' * new type:', type)
    return type, all_reasons

def determine_gender(page):
    all_genders = set()
    all_reasons = []
    genders, reason = determine_gender_from_firstname(page)
    if genders:
        all_genders.update(genders)
        all_reasons.append(reason)
    genders, reason = determine_gender_from_categories(page)
    if genders:
        all_genders.update(genders)
        all_reasons.append(reason)
    genders, reason = determine_gender_from_text(page)
    if genders:
        all_genders.update(genders)
        all_reasons.append(reason)
    if not all_reasons:
        colored_out(bcolors.WARNING, ' * not enough sources for genders')
        return None, []
    if len(all_genders) > 1:
        colored_out(bcolors.FAIL, ' * conflicting genders', all_genders, all_reasons)
        return None, []
    gender = list(all_genders)[0]
    colored_out(bcolors.OKGREEN, ' * new gender:', gender)
    return gender, all_reasons

      JOIN l_recording_work lrw ON recording.id = lrw.entity0
      JOIN l_artist_work law ON lrw.entity1 = law.entity1
     WHERE acn.artist = %s
    UNION
    -- Select artists of recordings of works for this artist (i.e. performers of works this artist wrote)
    SELECT acn.artist AS artist
      FROM artist_credit_name acn
      JOIN recording ON acn.artist_credit = recording.artist_credit
      JOIN l_recording_work lrw ON recording.id = lrw.entity0
      JOIN l_artist_work law ON lrw.entity1 = law.entity1
     WHERE law.entity0 = %s
)
"""

for artist in db.execute(query, query_params):
    colored_out(bcolors.OKBLUE, 'Looking up artist "%s" http://musicbrainz.org/artist/%s' % (artist['name'], artist['gid']))
    matches = wps.query(escape_query(artist['name']), defType='dismax', qf='name', rows=50).results
    last_wp_request = time.time()
    for match in matches:
        title = match['name']
        if title.endswith('album)') or title.endswith('song)'):
            continue
        if mangle_name(re.sub(' \(.+\)$', '', title)) != mangle_name(artist['name']) and mangle_name(title) != mangle_name(artist['name']):
            continue
        delay = time.time() - last_wp_request
        if delay < 1.0:
            time.sleep(1.0 - delay)
        last_wp_request = time.time()
        wikipage = WikiPage.fetch('https://%s.wikipedia.org/wiki/%s' % (wp_lang, title))
        page_orig = wikipage.text
        if not page_orig:

    )
SELECT a.id, a.gid, a.name, aws.shs_url, aws.work_id, aws.work_gid, b.processed
FROM artists_wo_shs aws
JOIN s_artist a ON aws.artist_id = a.id
LEFT JOIN bot_shs_link_artist b ON a.gid = b.artist
ORDER BY b.processed NULLS FIRST, a.id
LIMIT 1000
"""

seen_artists = set()
matched_artists = set()
for artist in db.execute(query):
    if artist['gid'] in matched_artists:
        continue

    colored_out(bcolors.OKBLUE, 'Looking up artist "%s" http://musicbrainz.org/artist/%s' % (artist['name'], artist['gid']))

    m = re.match(r'http://www.secondhandsongs.com/work/([0-9]+)', artist['shs_url'])
    if m:
        shs_work = shs.lookup_work(int(m.group(1)))
    else:
        continue
    
    artist_uri = None
    shs_artists = []
    # credits of actual work
    if 'credits' in shs_work and len(shs_work['credits']) > 0:
        shs_artists.extend(shs_work['credits'])
    # credits of original work
    if 'originalCredits' in shs_work and len(shs_work['originalCredits']) > 0:
        shs_artists.extend(shs_work['originalCredits'])

def main():
    seen = set()
    matched = set()
    for artist in db.execute(query):
        if artist['gid'] in matched:
            continue

        colored_out(bcolors.OKBLUE, 'Looking up artist "%s" http://musicbrainz.org/artist/%s' % (artist['name'], artist['gid']))
        out(' * wiki:', artist['wp_url'])

        page = WikiPage.fetch(artist['wp_url'], False)
        identifiers = determine_authority_identifiers(page)
        if 'VIAF' in identifiers:
            if not isinstance(identifiers['VIAF'], basestring):
                colored_out(bcolors.FAIL, ' * multiple VIAF found: %s' % ', '.join(identifiers['VIAF']))
            elif identifiers['VIAF'] == '' or identifiers['VIAF'] is None:
                colored_out(bcolors.FAIL, ' * invalid empty VIAF found')
            else:
                viaf_url = 'http://viaf.org/viaf/%s' % identifiers['VIAF']
                edit_note = 'From %s' % (artist['wp_url'],)
                colored_out(bcolors.OKGREEN, ' * found VIAF:', viaf_url)
                # Check if this VIAF has not been deleted
                skip = False
                try:
                    resp, content = httplib2.Http().request(viaf_url)
                except socket.error:
                    colored_out(bcolors.FAIL, ' * timeout!')
                    skip = True
                deleted_message = 'abandonedViafRecord'
                if skip == False and (resp.status == '404' or deleted_message in content):
                    colored_out(bcolors.FAIL, ' * deleted VIAF!')
                    skip = True
                if skip == False:
                    time.sleep(3)
                    out(' * edit note:', edit_note.replace('\n', ' '))
                    mb.add_url('artist', artist['gid'], str(VIAF_RELATIONSHIP_TYPES['artist']), viaf_url, edit_note)
                    matched.add(artist['gid'])

        if artist['processed'] is None and artist['gid'] not in seen:
            db.execute("INSERT INTO bot_wp_artist_viaf (gid, lang) VALUES (%s, %s)", (artist['gid'], page.lang))
        else:
            db.execute("UPDATE bot_wp_artist_viaf SET processed = now() WHERE (gid, lang) = (%s, %s)", (artist['gid'], page.lang))
        seen.add(artist['gid'])

        AND NOT EXISTS (SELECT 1 FROM l_recording_work lrw2 WHERE lrw2.entity0 = r.id AND lrw2.entity1 <> lrw.entity1)
    ORDER BY r.artist_credit
    LIMIT 750
"""

date_re = re.compile(r'live, (\d{4})(?:-(\d{2}))?(?:-(\d{2}))?:', re.I)
for recording in db.execute(query):

    m = re.match(date_re, recording['comment'])
    if m is None:
        continue

    date = {'year': int(m.group(1))}

    if m.group(2) is not None:
        date['month'] = int(m.group(2))
    if m.group(3) is not None:
        date['day'] = int(m.group(3))
  
    colored_out(bcolors.OKBLUE, 'Setting performance relationships dates of http://musicbrainz.org/recording/%s "%s (%s)"' % (recording['r_gid'], recording['name'], recording['comment']))

    attributes = {}
    edit_note = 'Setting relationship dates from recording comment: "%s"' % recording['comment']
    colored_out(bcolors.NONE, " * new date:", date)
    
    entity0 = {'type': 'recording', 'gid': recording['r_gid']}
    entity1 = {'type': 'work', 'gid': recording['w_gid']}

    time.sleep(2)
    mb.edit_relationship(recording['rel_id'], entity0, entity1, recording['link_type'], attributes, date, date, edit_note, True)

def main(ENTITY_TYPE):

    entity_type_table = ENTITY_TYPE.replace("-", "_")
    url_relationship_table = "l_%s_url" % entity_type_table if ENTITY_TYPE != "work" else "l_url_%s" % entity_type_table
    main_entity_entity_point = "entity0" if ENTITY_TYPE != "work" else "entity1"
    url_entity_point = "entity1" if ENTITY_TYPE != "work" else "entity0"

    query = (
        """
    WITH
        entities_wo_wikidata AS (
            SELECT DISTINCT e.id AS entity_id, e.gid AS entity_gid, u.url AS wp_url
            FROM """
        + entity_type_table
        + """ e
                JOIN """
        + url_relationship_table
        + """ l ON l."""
        + main_entity_entity_point
        + """ = e.id AND l.link IN (SELECT id FROM link WHERE link_type = """
        + str(WIKIPEDIA_RELATIONSHIP_TYPES[ENTITY_TYPE])
        + """)
                JOIN url u ON u.id = l."""
        + url_entity_point
        + """ AND u.url LIKE 'http://%%.wikipedia.org/wiki/%%' AND substring(u.url from 8 for 2) IN ('en', 'fr')
            WHERE 
                /* No existing WikiData relationship for this entity */
                NOT EXISTS (SELECT 1 FROM """
        + url_relationship_table
        + """ ol WHERE ol."""
        + main_entity_entity_point
        + """ = e.id AND ol.link IN (SELECT id FROM link WHERE link_type = """
        + str(WIKIDATA_RELATIONSHIP_TYPES[ENTITY_TYPE])
        + """))
                /* WP link should only be linked to this entity */
                AND NOT EXISTS (SELECT 1 FROM """
        + url_relationship_table
        + """ ol WHERE ol."""
        + url_entity_point
        + """ = u.id AND ol."""
        + main_entity_entity_point
        + """ <> e.id)
                AND l.edits_pending = 0
        )
    SELECT e.id, e.gid, e.name, ewf.wp_url, b.processed
    FROM entities_wo_wikidata ewf
    JOIN s_"""
        + entity_type_table
        + """ e ON ewf.entity_id = e.id
    LEFT JOIN bot_wp_wikidata_links b ON e.gid = b.gid AND b.lang = substring(ewf.wp_url from 8 for 2)
    ORDER BY b.processed NULLS FIRST, e.id
    LIMIT 250
    """
    )

    seen = set()
    matched = set()
    for entity in db.execute(query):
        if entity["gid"] in matched:
            continue

        colored_out(
            bcolors.OKBLUE,
            'Looking up entity "%s" http://musicbrainz.org/%s/%s' % (entity["name"], ENTITY_TYPE, entity["gid"]),
        )
        out(" * wiki:", entity["wp_url"])

        page = WikiPage.fetch(entity["wp_url"], False)
        if page.wikidata_id:
            wikidata_url = "http://www.wikidata.org/wiki/%s" % page.wikidata_id.upper()
            edit_note = "From %s" % (entity["wp_url"],)
            colored_out(bcolors.OKGREEN, " * found WikiData identifier:", wikidata_url)
            time.sleep(3)
            out(" * edit note:", edit_note.replace("\n", " "))
            mb.add_url(
                ENTITY_TYPE.replace("-", "_"),
                entity["gid"],
                str(WIKIDATA_RELATIONSHIP_TYPES[ENTITY_TYPE]),
                wikidata_url,
                edit_note,
                True,
            )
            matched.add(entity["gid"])

        if entity["processed"] is None and entity["gid"] not in seen:
            db.execute("INSERT INTO bot_wp_wikidata_links (gid, lang) VALUES (%s, %s)", (entity["gid"], page.lang))
        else:
            db.execute(
                "UPDATE bot_wp_wikidata_links SET processed = now() WHERE (gid, lang) = (%s, %s)",
                (entity["gid"], page.lang),
            )
        seen.add(entity["gid"])

def main():
    seen = set()
    for artist in db.execute(query):
        if artist['id'] in seen:
            continue
        seen.add(artist['id'])
        colored_out(bcolors.OKBLUE, 'Looking up artist "%s" http://musicbrainz.org/artist/%s' % (artist['name'], artist['gid']))
        out(' * wiki:', artist['url'])

        artist = dict(artist)
        update = set()
        reasons = []

        page = WikiPage.fetch(artist['url'])

        if not artist['country']:
            country, country_reasons = determine_country(page)
            if country:
                country_id = country_ids[country]
                artist['country'] = country_id
                update.add('country')
                reasons.append(('COUNTRY', country_reasons))

        if not artist['type']:
            type, type_reasons = determine_type(page)
            if type:
                type_id = artist_type_ids[type]
                artist['type'] = type_id
                update.add('type')
                reasons.append(('TYPE', type_reasons))

        if not artist['gender'] and artist['type'] == 1:
            gender, gender_reasons = determine_gender(page)
            if gender:
                gender_id = gender_ids[gender]
                artist['gender'] = gender_id
                update.add('gender')
                reasons.append(('GENDER', gender_reasons))

        is_performance_name = False
        if artist['type'] == 1 and CHECK_PERFORMANCE_NAME:
            is_performance_name = db.execute(performance_name_query, artist['id']).scalar() > 0
            out(" * checking for performance name", is_performance_name)

        if not artist['begin_date_year']:
            begin_date, begin_date_reasons = determine_begin_date(artist, page, is_performance_name)
            if begin_date['year']:
                colored_out(bcolors.OKGREEN, " * new begin date:", begin_date)
                artist['begin_date_year'] = begin_date['year']
                artist['begin_date_month'] = begin_date['month']
                artist['begin_date_day'] = begin_date['day']
                update.add('begin_date')
                reasons.append(('BEGIN DATE', begin_date_reasons))
        if not artist['end_date_year']:
            end_date, end_date_reasons = determine_end_date(artist, page, is_performance_name)
            if end_date['year']:
                colored_out(bcolors.OKGREEN, " * new end date:", end_date)
                artist['end_date_year'] = end_date['year']
                artist['end_date_month'] = end_date['month']
                artist['end_date_day'] = end_date['day']
                update.add('end_date')
                reasons.append(('END DATE', end_date_reasons))

        if update:
            edit_note = 'From %s' % (artist['url'],)
            for field, reason in reasons:
                edit_note += '\n\n%s:\n%s' % (field, ' '.join(reason))
            out(' * edit note:', edit_note.replace('\n', ' '))
            time.sleep(10)
            mb.edit_artist(artist, update, edit_note)

        db.execute("INSERT INTO bot_wp_artist_data (gid, lang) VALUES (%s, %s)", (artist['gid'], wp_lang))
        out()

DISCOGS_MB_FORMATS_MAPPING = {
    "Vinyl": 7,
    '12"': 31,
    '10"': 30,
    '7"': 29,
    "CD": 1,
    "CDr": 33,
    "Cassette": 8,
    "DigitalMedia": 12,
}

for medium in db.execute(query):
    colored_out(
        bcolors.OKBLUE,
        'Looking up medium #%s of release "%s" by "%s" http://musicbrainz.org/release/%s'
        % (medium["position"], medium["name"], medium["ac_name"], medium["gid"]),
    )

    m = re.match(r"http://www.discogs.com/release/([0-9]+)", medium["discogs_url"])
    if m:
        discogs_release = discogs.Release(int(m.group(1)))

    discogs_format = discogs_get_medium_format(discogs_release, medium["position"])
    if discogs_format:
        colored_out(bcolors.HEADER, " * using %s, found format: %s" % (medium["discogs_url"], discogs_format))
        edit_note = "Setting medium format from attached Discogs link (%s)" % medium["discogs_url"]
        out(" * edit note: %s" % (edit_note,))
        mb.set_release_medium_format(
            medium["gid"],
            medium["position"],