Python TablePatcher.process方法代码示例

# 需要导入模块: from mptracker.patcher import TablePatcher [as 别名]
# 或者: from mptracker.patcher.TablePatcher import process [as 别名]
def update_person_xls():
    """ Update person contact data from csv"""
    from mptracker.scraper.person_xls import read_person_contact

    mandate_lookup = models.MandateLookup()

    people_data = []
    mandate_patcher = TablePatcher(models.Mandate,
                                   models.db.session,
                                   key_columns=['year', 'cdep_number'])
    with mandate_patcher.process() as add:
        for record in read_person_contact(MINORITIES_CSV_KEY):
            mandate = mandate_lookup.find(record.pop('name'), record['year'],
                                          record['cdep_number'])
            person_data = record.pop('person_data')
            person_data['id'] = mandate.person_id
            people_data.append(person_data)
            add(record)

    person_patcher = TablePatcher(models.Person,
                                  models.db.session,
                                  key_columns=['id'])
    with person_patcher.process() as add:
        for person_data in people_data:
            add(person_data)

    models.db.session.commit()

# 需要导入模块: from mptracker.patcher import TablePatcher [as 别名]
# 或者: from mptracker.patcher.TablePatcher import process [as 别名]
def get_vote_controversy(no_commit=False):
    controversy_patcher = TablePatcher(
        models.VotingSessionControversy,
        models.db.session,
        key_columns=['voting_session_id'],
    )

    with controversy_patcher.process(remove=True) as add_controversy:
        for line in get_gdrive_csv(CONTROVERSY_CSV_KEY):
            add_controversy({
                'title': line['title'],
                'status': line['status'],
                'reason': line['motive'],
                'vote_meaning_yes': line['info_da'],
                'vote_meaning_no': line['info_nu'],
                'press_links': line['link_presa'],
                'voting_session_id': line['mptracker_url'].split('/votes/')[1],
            })

    if no_commit:
        logger.warn("Rolling back the transaction")
        models.db.session.rollback()

    else:
        models.db.session.commit()

# 需要导入模块: from mptracker.patcher import TablePatcher [as 别名]
# 或者: from mptracker.patcher.TablePatcher import process [as 别名]
def committees(
    cache_name=None,
    throttle=None,
    no_commit=False,
):
    from mptracker.scraper.committees import CommitteeScraper

    patcher = TablePatcher(
        models.MpCommittee,
        models.db.session,
        key_columns=['chamber_id', 'cdep_id'],
    )

    http_session = create_session(
        cache_name=cache_name,
        throttle=throttle and float(throttle),
    )
    scraper = CommitteeScraper(http_session)
    with patcher.process(autoflush=1000, remove=True) as add:
        for committee in scraper.fetch_committees():
            add(committee.as_dict(['chamber_id', 'cdep_id', 'name']))

    if no_commit:
        logger.warn("Rolling back the transaction")
        models.db.session.rollback()

    else:
        models.db.session.commit()

# 需要导入模块: from mptracker.patcher import TablePatcher [as 别名]
# 或者: from mptracker.patcher.TablePatcher import process [as 别名]
def get_committee_policy():
    patcher = TablePatcher(
        models.MpCommittee,
        models.db.session,
        key_columns=['id'],
    )

    with patcher.process() as update_committee:
        for row in get_gdrive_csv(COMMITTEE_POLICY_CSV_KEY):
            slug = row['policy']

            policy_id = None
            if slug:
                policy = models.PolicyDomain.query.filter_by(slug=slug).first()
                if policy is None:
                    logger.warn("Unknown policy domain %r", slug)
                else:
                    policy_id = policy.id

            update_committee(
                dict(id=row['id'], policy_domain_id=policy_id),
                create=False
            )

    models.db.session.commit()

# 需要导入模块: from mptracker.patcher import TablePatcher [as 别名]
# 或者: from mptracker.patcher.TablePatcher import process [as 别名]
def get_proposal_controversy():
    """ Update proposal controversies from csv"""

    def extract_proposal(url):
        return url[url.rfind('/') + 1:]

    controversy_patcher = TablePatcher(models.ProposalControversy,
                                       models.db.session,
                                       key_columns=['proposal_id'])
    with controversy_patcher.process(remove=True) as add:
        for row in get_gdrive_csv(PROPOSAL_CONTROVERSY_CSV_KEY):
            proposal_id = extract_proposal(row['Link MP Tracker'])
            if not proposal_id:
                continue
            assert models.Proposal.query.get(proposal_id)

            record = {
                'proposal_id': proposal_id,
                'title': row['Titlu'],
                'reason': row['Motive controversa'],
                'press_links': row['Link presa'],
            }
            add(record)

    models.db.session.commit()

# 需要导入模块: from mptracker.patcher import TablePatcher [as 别名]
# 或者: from mptracker.patcher.TablePatcher import process [as 别名]
def assets(file_path, no_commit=False):
    from mptracker.scraper.assets import parse_assets
    from mptracker.nlp import normalize

    asset_patcher = TablePatcher(
        models.AssetStatement,
        models.db.session,
        key_columns=['person_id', 'date'],
    )

    people_map = {
        normalize(person.name): person.id
        for person in (
            models.Person.query
            .join(models.Person.mandates)
            .filter_by(year=2012)
        )
    }

    with asset_patcher.process(remove=True) as add_asset:
        for record in parse_assets(file_path):
            person_name = normalize(record.pop('person_name'))
            person_id = people_map[person_name]
            del record['constituency']
            del record['county']
            res = add_asset({
                'person_id': person_id,
                'date': date(2012, 11, 1),
                'raw_data': record,
                'net_worth_eur': (
                    record['acct_value']['TOTAL_EUR']
                    - record['debt_value']['TOTAL_EUR']
                    + record['invest_value']['TOTAL_EUR']
                    + record['valuables_value']['TOTAL_EUR']
                ),
                'land_agri_area': record['land_agri_area'],
                'land_city_area': record['land_city_area'],
                'realty_count': (
                    record['realty_apartment_count'] +
                    record['realty_business_count'] +
                    record['realty_house_count']
                ),
                'vehicle_count': record['vehicle_count'],
                'year_income_eur': (
                    record['family_income_value']['TOTAL_EUR'] +
                    record['gift_value']['TOTAL_EUR'] +
                    record['sales_value']['TOTAL_EUR']
                ),
            })

    if no_commit:
        logger.warn("Rolling back the transaction")
        models.db.session.rollback()

    else:
        models.db.session.commit()

# 需要导入模块: from mptracker.patcher import TablePatcher [as 别名]
# 或者: from mptracker.patcher.TablePatcher import process [as 别名]
def position():
    name_search = models.NameSearch(
        models.Person.query
        .join(models.Mandate)
        .filter(models.Mandate.year == 2012)
        .all()
    )

    position_patcher = TablePatcher(
        models.Position,
        models.db.session,
        key_columns=['person_id', 'interval', 'title'],
    )

    with position_patcher.process(remove=True) as add_position:
        for row in get_gdrive_csv(POSITION_PONTA2_CSV_KEY):
            if row['temporary'].strip():
                continue

            name = row['name'].strip()
            matches = name_search.find(name)

            if len(matches) == 1:
                [person] = matches
                interval = parse_interval(row['start_date'], row['end_date'])
                add_position({
                    'person_id': person.id,
                    'interval': interval,
                    'title': row['title'],
                    'url': row['url'] or None,
                })

            elif len(matches) > 1:
                logger.warn("Multiple matches for %r", name)

            else:
                logger.warn("No matches for %r", name)

        for row in get_gdrive_csv(POSITION_BIROU_CDEP_CSV_KEY):
            name = row['name'].strip()
            matches = name_search.find(name)

            assert len(matches) == 1, \
                "Expected a single match for %r, got %r" % (name, matches)

            [person] = matches
            add_position({
                'person_id': person.id,
                'interval': parse_interval(row['start_date'], row['end_date']),
                'title': row['title'] + ", Biroul Permanent",
            })

    models.db.session.commit()

# 需要导入模块: from mptracker.patcher import TablePatcher [as 别名]
# 或者: from mptracker.patcher.TablePatcher import process [as 别名]
def policy_domain():
    patcher = TablePatcher(
        models.PolicyDomain,
        models.db.session,
        key_columns=['slug'],
    )

    with patcher.process(remove=True) as add_policy_domain:
        for row in get_gdrive_csv(POLICY_DOMAIN_CSV_KEY):
            add_policy_domain(row)

    models.db.session.commit()

# 需要导入模块: from mptracker.patcher import TablePatcher [as 别名]
# 或者: from mptracker.patcher.TablePatcher import process [as 别名]
def stop_words():
    patcher = TablePatcher(
        models.Stopword,
        models.db.session,
        key_columns=['id'],
    )

    with patcher.process(remove=True) as add_stop_word:
        for row in get_gdrive_csv(STOP_WORDS_CSV_KEY):
            add_stop_word(row)

    models.db.session.commit()

# 需要导入模块: from mptracker.patcher import TablePatcher [as 别名]
# 或者: from mptracker.patcher.TablePatcher import process [as 别名]
def stop_words():
    from mptracker.nlp import normalize_to_ascii
    patcher = TablePatcher(
        models.Stopword,
        models.db.session,
        key_columns=['id'],
    )

    with patcher.process(remove=True) as add_stop_word:
        for row in get_gdrive_csv(STOP_WORDS_CSV_KEY):
            add_stop_word({'id': normalize_to_ascii(row['id'])})

    models.db.session.commit()

# 需要导入模块: from mptracker.patcher import TablePatcher [as 别名]
# 或者: from mptracker.patcher.TablePatcher import process [as 别名]
def get_member_count():
    patcher = TablePatcher(
        models.MemberCount,
        models.db.session,
        key_columns=['short_name', 'year'],
    )

    with patcher.process(remove=True) as add_member_count:
        for row in get_gdrive_csv(MEMBER_COUNT_CSV_KEY):
            short_name = row.pop('')
            for year, count in row.items():
                add_member_count({
                    'short_name': short_name,
                    'year': int(year),
                    'count': int(count),
                })

    models.db.session.commit()

# 需要导入模块: from mptracker.patcher import TablePatcher [as 别名]
# 或者: from mptracker.patcher.TablePatcher import process [as 别名]
def cabinet_party():
    patcher = TablePatcher(
        models.CabinetMembership,
        models.db.session,
        key_columns=['mp_group_id', 'interval'],
    )

    group_by_code = {g.short_name: g for g in models.MpGroup.query}

    with patcher.process(remove=True) as add_membership:
        for row in get_gdrive_csv(CABINET_PARTY_CSV_KEY):
            assert row['legislature'] == '2012'
            group = group_by_code[row['code']]
            add_membership({
                'mp_group_id': group.id,
                'interval': parse_interval(row['start_date'], row['end_date']),
            })

    models.db.session.commit()

# 需要导入模块: from mptracker.patcher import TablePatcher [as 别名]
# 或者: from mptracker.patcher.TablePatcher import process [as 别名]
def people(
    year='2012',
    cache_name=None,
    throttle=None,
    no_commit=False,
    add_people=False,
):
    from mptracker.scraper.people import MandateScraper

    http_session = create_session(
        cache_name=cache_name,
        throttle=throttle and float(throttle),
    )
    mandate_scraper = MandateScraper(http_session)

    mandate_patcher = TablePatcher(
        models.Mandate,
        models.db.session,
        key_columns=['year', 'cdep_number'],
    )

    person_patcher = TablePatcher(
        models.Person,
        models.db.session,
        key_columns=['id'],
    )

    new_people = 0
    chamber_by_slug = {c.slug: c for c in models.Chamber.query}

    with mandate_patcher.process() as add_mandate, \
         person_patcher.process() as add_person:
        for mandate in mandate_scraper.fetch(year):
            row = mandate.as_dict([
                'year',
                'cdep_number',
                'minority',
                'college',
                'constituency',
                'picture_url',
            ])
            assert mandate.chamber_number == 2
            row['chamber_id'] = chamber_by_slug['cdep'].id
            if year == '2012':
                end_date = mandate.end_date or date.max
                row['interval'] = DateRange(TERM_2012_START, end_date)

            person = (
                models.Person.query
                    .filter_by(name=mandate.person_name)
                    .first())

            if person is None:
                if add_people:
                    person = models.Person(name=mandate.person_name)
                    models.db.session.add(person)
                    models.db.session.flush()
                    new_people += 1

                else:
                    raise RuntimeError("Can't find person named %r"
                                       % mandate.person_name)

            assert not add_person({
                'id': person.id,
                'first_name': mandate.person_first_name,
                'last_name': mandate.person_last_name,
            }).is_new

            row['person_id'] = person.id

            if not mandate.minority:
                county = (
                    models.County.query
                        .filter_by(name=mandate.county_name)
                        .first())
                if county is None:
                    raise RuntimeError("Can't match county name %r"
                                       % mandate.county_name)
                row['county'] = county

            add_mandate(row)

    if new_people:
        logger.info("%d new people", new_people)

    if no_commit:
        logger.warn("Rolling back the transaction")
        models.db.session.rollback()

    else:
        models.db.session.commit()

# 需要导入模块: from mptracker.patcher import TablePatcher [as 别名]
# 或者: from mptracker.patcher.TablePatcher import process [as 别名]
def controversy():
    old_voting_sessions = set(
        models.VotingSession.query
        .filter(models.VotingSession.controversy_id != None)
        .all()
    )

    controversy_map = {}

    for line in get_gdrive_csv(CONTROVERSY_CSV_KEY):
        cdeppk = url_args(line['link']).get('idv', type=int)
        slug = line['slug']
        if slug not in controversy_map:
            controversy_map[slug] = {
                'data': {
                    'slug': slug,
                    'title': line['title'],
                },
                'voting_session_rows': [],
            }

        voting_session = (
            models.VotingSession.query
            .filter_by(cdeppk=cdeppk)
            .first()
        )
        controversy_map[slug]['voting_session_rows'].append(voting_session)

    controversy_patcher = TablePatcher(
        models.Controversy,
        models.db.session,
        key_columns=['slug'],
    )

    with controversy_patcher.process(remove=True) as add_controversy:
        for controversy in controversy_map.values():
            result = add_controversy(controversy['data'])
            controversy['row'] = result.row

    models.db.session.flush()

    voting_session_patcher = TablePatcher(
        models.VotingSession,
        models.db.session,
        key_columns=['id'],
    )

    new_voting_sessions = set()

    with voting_session_patcher.process() as add_voting_session:
        for controversy in controversy_map.values():
            for voting_session in controversy['voting_session_rows']:
                data = {
                    'id': voting_session.id,
                    'controversy_id': controversy['row'].id,
                }
                add_voting_session(data, create=False)
                new_voting_sessions.add(voting_session)

        for voting_session in old_voting_sessions - new_voting_sessions:
            add_voting_session({
                'id': voting_session.id,
                'controversy_id': None,
            })

    models.db.session.commit()

# 需要导入模块: from mptracker.patcher import TablePatcher [as 别名]
# 或者: from mptracker.patcher.TablePatcher import process [as 别名]
def groups(
        cache_name=None,
        throttle=None,
        no_commit=False,
        year='2012',
        ):
    year = int(year)

    from mptracker.scraper.groups import GroupScraper, Interval

    http_session = create_session(cache_name=cache_name,
                                  throttle=throttle and float(throttle))
    group_scraper = GroupScraper(http_session)

    mandate_lookup = models.MandateLookup()
    mandate_intervals = defaultdict(list)
    term_interval = TERM_INTERVAL[year]

    groups = list(group_scraper.fetch(year))
    independents = groups[0]
    assert independents.is_independent
    for group in groups[1:] + [independents]:
        for member in group.current_members + group.former_members:
            (myear, chamber, number) = member.mp_ident
            assert chamber == 2
            mandate = mandate_lookup.find(member.mp_name, myear, number)
            interval_list = mandate_intervals[mandate]

            interval = member.get_interval()
            if interval.start is None:
                interval = interval._replace(start=term_interval.lower)

            if interval.end is None:
                interval = interval._replace(end=term_interval.upper)

            if group.is_independent:
                if interval_list:
                    start = interval_list[-1].end
                    interval = interval._replace(start=start)

            interval_list.append(interval)
            interval_list.sort(key=lambda i: i[0])

    for mandate, interval_list in mandate_intervals.items():
        # make sure interval_list are continuous
        new_intervals = []
        for interval_one, interval_two in \
            zip(interval_list[:-1], interval_list[1:]):

            assert interval_one.start < interval_one.end
            if interval_one.end < interval_two.start:
                interval = Interval(
                    start=interval_one.end,
                    end=interval_two.start,
                    group=independents,
                )
                new_intervals.append(interval)
            elif interval_one.end > interval_two.start:
                import pdb; pdb.set_trace()
                raise RuntimeError("Overlapping intervals")

        interval_list.extend(new_intervals)
        interval_list.sort()

        mandate_end = mandate.interval.upper
        if mandate_end == date.max:
            mandate_end = None
        if interval_list[-1].end != mandate_end:
            logger.warn("Mandate %s ends at %s",
                        mandate, interval_list[-1].end)

    group_patcher = TablePatcher(
        models.MpGroup,
        models.db.session,
        key_columns=['short_name', 'year'],
    )

    with group_patcher.process(remove=True, filter={'year': year}) as add_group:
        for group in groups:
            record = group.as_dict(['name', 'short_name', 'year'])
            group.row = add_group(record).row

        models.db.session.flush()

    membership_patcher = TablePatcher(
        models.MpGroupMembership,
        models.db.session,
        key_columns=['mandate_id', 'mp_group_id', 'interval'],
    )

    current_membership_query = (
        models.db.session.query(models.MpGroupMembership.id)
        .join(models.MpGroupMembership.mandate)
        .filter_by(year=year)
    )

    remove_membership_ids = set(row.id for row in current_membership_query)
    with membership_patcher.process(autoflush=1000) as add_membership:
        for mandate, interval_list in mandate_intervals.items():
            for interval in interval_list:
#.........这里部分代码省略.........