Python Vote.add_source方法代码示例

# 需要导入模块: from fiftystates.scrape.votes import Vote [as 别名]
# 或者: from fiftystates.scrape.votes.Vote import add_source [as 别名]
    def scrape_votes(self, link, chamber, bill):
        with self.lxml_context(link) as votes_page:
            page_tables = votes_page.cssselect("table")
            votes_table = page_tables[0]
            votes_elements = votes_table.cssselect("td")
            # Eliminate table headings and unnecessary element
            votes_elements = votes_elements[3 : len(votes_elements)]
            ve = self.grouper(5, votes_elements)
            for actor, date, name_and_text, name, text in ve:
                if "cow" in text.text_content() or "COW" in text.text_content():
                    continue
                vote_date = dt.datetime.strptime(date.text_content(), "%m/%d/%Y")
                motion_and_votes = text.text_content().lstrip("FINAL VOTE - ")
                motion, sep, votes = motion_and_votes.partition(".")
                if "passed" in votes:
                    passed = True
                else:
                    passed = False

                votes_match = re.search("([0-9]+)-([0-9]+)-?([0-9]+)?", votes)
                yes_count = votes_match.group(1)
                no_count = votes_match.group(2)
                other_count = votes_match.group(3)

                if other_count == None:
                    other_count = 0

                vote = Vote(chamber, vote_date, motion, passed, yes_count, no_count, other_count)
                vote.add_source(link)
                bill.add_vote(vote)

# 需要导入模块: from fiftystates.scrape.votes import Vote [as 别名]
# 或者: from fiftystates.scrape.votes.Vote import add_source [as 别名]
    def scrape_vote(self, bill, name, url):
        match = re.match('^(Senate|House) Vote on [^,]*,(.*)$', name)

        if not match:
            return

        chamber = {'Senate': 'upper', 'House': 'lower'}[match.group(1)]
        motion = match.group(2).strip()

        if motion.startswith('FINAL PASSAGE'):
            type = 'passage'
        elif motion.startswith('AMENDMENT'):
            type = 'amendment'
        elif 'ON 3RD READINT' in motion:
            type = 'reading:3'
        else:
            type = 'other'

        vote = Vote(chamber, None, motion, None,
                    None, None, None)
        vote['type'] = type
        vote.add_source(url)

        with self.urlopen(url) as text:
            (fd, temp_path) = tempfile.mkstemp()
            with os.fdopen(fd, 'wb') as w:
                w.write(text)
            html = pdf_to_lxml(temp_path)
            os.remove(temp_path)

            vote_type = None
            total_re = re.compile('^Total--(\d+)$')
            body = html.xpath('string(/html/body)')
            for line in body.replace(u'\xa0', '\n').split('\n'):
                line = line.replace(' ', '').strip()
                if not line:
                    continue

                if line in ('YEAS', 'NAYS', 'ABSENT'):
                    vote_type = {'YEAS': 'yes', 'NAYS': 'no',
                                 'ABSENT': 'other'}[line]
                elif vote_type:
                    match = total_re.match(line)
                    if match:
                        vote['%s_count' % vote_type] = int(match.group(1))
                    elif vote_type == 'yes':
                        vote.yes(line)
                    elif vote_type == 'no':
                        vote.no(line)
                    elif vote_type == 'other':
                        vote.other(line)

        # The PDFs oddly don't say whether a vote passed or failed.
        # Hopefully passage just requires yes_votes > not_yes_votes
        if vote['yes_count'] > (vote['no_count'] + vote['other_count']):
            vote['passed'] = True
        else:
            vote['passed'] = False

        bill.add_vote(vote)

# 需要导入模块: from fiftystates.scrape.votes import Vote [as 别名]
# 或者: from fiftystates.scrape.votes.Vote import add_source [as 别名]
    def parse_vote(self, bill, action, act_chamber, act_date, url):
        url = "http://www.legis.state.ak.us/basis/%s" % url
        info_page = self.soup_parser(self.urlopen(url))

        tally = re.findall('Y(\d+) N(\d+)\s*(?:\w(\d+))*\s*(?:\w(\d+))*'
                           '\s*(?:\w(\d+))*', action)[0]
        yes, no, o1, o2, o3 = map(lambda x: 0 if x == '' else int(x), tally)
        yes, no, other = int(yes), int(no), (int(o1) + int(o2) + int(o3))

        votes = info_page.findAll('pre', text=re.compile('Yeas'),
                                  limit=1)[0].split('\n\n')

        motion = info_page.findAll(text=re.compile('The question being'))[0]
        motion = re.findall('The question being:\s*"(.*)\?"',
                            motion, re.DOTALL)[0].replace('\n', ' ')

        vote = Vote(act_chamber, act_date, motion, yes > no, yes, no, other)

        for vote_list in votes:
            vote_type = False
            if vote_list.startswith('Yeas: '):
                vote_list, vote_type = vote_list[6:], vote.yes
            elif vote_list.startswith('Nays: '):
                vote_list, vote_type = vote_list[6:], vote.no
            elif vote_list.startswith('Excused: '):
                vote_list, vote_type = vote_list[9:], vote.other
            elif vote_list.startswith('Absent: '):
                vote_list, vote_type = vote_list[9:], vote.other
            if vote_type:
                for name in vote_list.split(','):
                    vote_type(name.strip())

        vote.add_source(url)
        return vote

# 需要导入模块: from fiftystates.scrape.votes import Vote [as 别名]
# 或者: from fiftystates.scrape.votes.Vote import add_source [as 别名]
    def parse_vote_new(self, bill, chamber, url):
        vote_page = BeautifulSoup(self.urlopen(url))
        table = vote_page.table
        info_row = table.findAll('tr')[1]

        date = info_row.td.contents[0]
        date = dt.datetime.strptime(date, '%m/%d/%Y')
        motion = info_row.findAll('td')[1].contents[0]
        yes_count = int(info_row.findAll('td')[2].contents[0])
        no_count = int(info_row.findAll('td')[3].contents[0])
        abs_count = int(info_row.findAll('td')[4].contents[0])
        passed = info_row.findAll('td')[5].contents[0] == 'Pass'

        vote = Vote(chamber, date, motion, passed,
                    yes_count, no_count, abs_count)
        vote.add_source(url)

        for tr in table.findAll('tr')[3:]:
            if len(tr.findAll('td')) != 2:
                continue

            name = tr.td.contents[0].split(' of')[0]
            type = tr.findAll('td')[1].contents[0]
            if type.startswith('Yea'):
                vote.yes(name)
            elif type.startswith('Nay'):
                vote.no(name)
            else:
                vote.other(name)

        bill.add_vote(vote)

# 需要导入模块: from fiftystates.scrape.votes import Vote [as 别名]
# 或者: from fiftystates.scrape.votes.Vote import add_source [as 别名]
    def add_vote(self, bill, chamber, date, line, text):
        votes = re.findall(r'Ayes (\d+)\, Noes (\d+)', text)
        (yes, no) = int(votes[0][0]), int(votes[0][1])

        vtype = 'other'
        for regex, type in motion_classifiers.iteritems():
            if re.match(regex, text):
                vtype = type
                break

        v = Vote(chamber, date, text, yes > no, yes, no, 0, type=vtype)

        # fetch the vote itself
        link = line.xpath('//a[contains(@href, "/votes/")]')
        if link:
            link = link[0].get('href')
            v.add_source(link)

            filename, resp = self.urlretrieve(link)

            if 'av' in link:
                self.add_house_votes(v, filename)
            elif 'sv' in link:
                self.add_senate_votes(v, filename)

        bill.add_vote(v)

# 需要导入模块: from fiftystates.scrape.votes import Vote [as 别名]
# 或者: from fiftystates.scrape.votes.Vote import add_source [as 别名]
    def scrape_vote(self, bill, date, url):
        with self.urlopen(url) as page:
            page = lxml.html.fromstring(page)

            header = page.xpath("string(//h4[contains(@id, 'hdVote')])")

            location = header.split(', ')[1]

            if location.startswith('House'):
                chamber = 'lower'
            elif location.startswith('Senate'):
                chamber = 'upper'
            else:
                raise ScrapeError("Bad chamber: %s" % chamber)

            committee = ' '.join(location.split(' ')[1:]).strip()
            if not committee or committee.startswith('of Representatives'):
                committee = None

            motion = ', '.join(header.split(', ')[2:]).strip()

            yes_count = int(
                page.xpath("string(//td[contains(@id, 'tdAyes')])"))
            no_count = int(
                page.xpath("string(//td[contains(@id, 'tdNays')])"))
            excused_count = int(
                page.xpath("string(//td[contains(@id, 'tdExcused')])"))
            absent_count = int(
                page.xpath("string(//td[contains(@id, 'tdAbsent')])"))
            other_count = excused_count + absent_count

            passed = yes_count > no_count

            if motion.startswith('Do Pass'):
                type = 'passage'
            elif motion == 'Concurred in amendments':
                type = 'amendment'
            elif motion == 'Veto override':
                type = 'veto_override'
            else:
                type = 'other'

            vote = Vote(chamber, date, motion, passed, yes_count, no_count,
                        other_count)
            vote['type'] = type

            if committee:
                vote['committee'] = committee

            vote.add_source(url)

            for td in page.xpath("//table[contains(@id, 'tblVotes')]/tr/td"):
                if td.text == 'Yea':
                    vote.yes(td.getprevious().text.strip())
                elif td.text == 'Nay':
                    vote.no(td.getprevious().text.strip())
                elif td.text in ('Excused', 'Absent'):
                    vote.other(td.getprevious().text.strip())

            bill.add_vote(vote)

# 需要导入模块: from fiftystates.scrape.votes import Vote [as 别名]
# 或者: from fiftystates.scrape.votes.Vote import add_source [as 别名]
    def scrape_votes(self, link, chamber, bill):
        with self.urlopen(link) as votes_page_html:
            votes_page = lxml.html.fromstring(votes_page_html)
            page_tables = votes_page.cssselect('table')
            votes_table = page_tables[0]
            votes_elements = votes_table.cssselect('td')
            # Eliminate table headings and unnecessary element
            votes_elements = votes_elements[3:len(votes_elements)]
            ve = grouper(5, votes_elements)
            for actor, date, name_and_text, name, text in ve:
                if 'cow' in text.text_content() or 'COW' in text.text_content():
                    continue
                vote_date = dt.datetime.strptime(date.text_content(), '%m/%d/%Y')
                motion_and_votes = text.text_content().lstrip('FINAL VOTE - ')
                motion, sep, votes = motion_and_votes.partition('.')
                if 'passed' in votes:
                    passed = True
                else:
                    passed = False

                votes_match = re.search('([0-9]+)-([0-9]+)-?([0-9]+)?', votes)
                yes_count = votes_match.group(1)
                no_count = votes_match.group(2)
                other_count = votes_match.group(3)

                if other_count == None:
                    other_count = 0

                vote = Vote(chamber, vote_date, motion, passed, \
                            yes_count, no_count, other_count)
                vote.add_source(link)
                bill.add_vote(vote)

# 需要导入模块: from fiftystates.scrape.votes import Vote [as 别名]
# 或者: from fiftystates.scrape.votes.Vote import add_source [as 别名]
    def scrape_vote(self, bill, name, url):
        match = re.match("^(Senate|House) Vote on [^,]*,(.*)$", name)

        if not match:
            return

        chamber = {"Senate": "upper", "House": "lower"}[match.group(1)]
        motion = match.group(2).strip()

        if motion.startswith("FINAL PASSAGE"):
            type = "passage"
        elif motion.startswith("AMENDMENT"):
            type = "amendment"
        elif "ON 3RD READINT" in motion:
            type = "reading:3"
        else:
            type = "other"

        vote = Vote(chamber, None, motion, None, None, None, None)
        vote["type"] = type
        vote.add_source(url)

        with self.urlopen(url) as text:
            (fd, temp_path) = tempfile.mkstemp()
            with os.fdopen(fd, "wb") as w:
                w.write(text)
            html = pdf_to_lxml(temp_path)
            os.remove(temp_path)

            vote_type = None
            total_re = re.compile("^Total--(\d+)$")
            body = html.xpath("string(/html/body)")
            for line in body.replace(u"\xa0", "\n").split("\n"):
                line = line.replace(" ", "").strip()
                if not line:
                    continue

                if line in ("YEAS", "NAYS", "ABSENT"):
                    vote_type = {"YEAS": "yes", "NAYS": "no", "ABSENT": "other"}[line]
                elif vote_type:
                    match = total_re.match(line)
                    if match:
                        vote["%s_count" % vote_type] = int(match.group(1))
                    elif vote_type == "yes":
                        vote.yes(line)
                    elif vote_type == "no":
                        vote.no(line)
                    elif vote_type == "other":
                        vote.other(line)

        # The PDFs oddly don't say whether a vote passed or failed.
        # Hopefully passage just requires yes_votes > not_yes_votes
        if vote["yes_count"] > (vote["no_count"] + vote["other_count"]):
            vote["passed"] = True
        else:
            vote["passed"] = False

        bill.add_vote(vote)

# 需要导入模块: from fiftystates.scrape.votes import Vote [as 别名]
# 或者: from fiftystates.scrape.votes.Vote import add_source [as 别名]
    def scrape_old_vote(self, url):
        vote_page = self.soup_parser(self.urlopen(url))

        header = vote_page.h3.contents[0]

        chamber_name = header.split(', ')[1]
        if chamber_name.startswith('House'):
            chamber = 'lower'
        else:
            chamber = 'upper'

        location = ' '.join(chamber_name.split(' ')[1:])
        if location.startswith('of Representatives'):
            location = ''

        motion = ', '.join(header.split(', ')[2:])

        def get_count(cell):
            if len(cell.contents) == 0:
                return 0
            else:
                return int(cell.contents[0])

        results_tbl = vote_page.findAll('table')[1]
        yes_count = get_count(results_tbl.findAll('td')[1])
        no_count = get_count(results_tbl.findAll('td')[3])
        excused_count = get_count(results_tbl.findAll('td')[5])
        absent_count = get_count(results_tbl.findAll('td')[7])
        other_count = excused_count + absent_count

        passed = yes_count > no_count

        vote = Vote(chamber, None, motion, passed,
                    yes_count, no_count,
                    other_count, excused_count=excused_count,
                    absent_count=absent_count,
                    location=location)
        vote.add_source(url)

        vote_tbl = vote_page.table
        for td in vote_tbl.findAll('td'):
            if td.contents[0] == 'Yea':
                vote.yes(td.findPrevious().contents[0])
            elif td.contents[0] == 'Nay':
                vote.no(td.findPrevious().contents[0])
            elif td.contents[0] in ['Excused', 'Absent']:
                vote.other(td.findPrevious().contents[0])

        return vote

# 需要导入模块: from fiftystates.scrape.votes import Vote [as 别名]
# 或者: from fiftystates.scrape.votes.Vote import add_source [as 别名]
    def scrape_votes(self, bill, sponsor, link):
        with self.urlopen(link) as page:
            page = lxml.html.fromstring(page)
            raw_vote_data = page.xpath("//span[@id='lblVoteData']")[0].text_content()
            raw_vote_data = raw_vote_data.strip().split('%s by %s - ' % (bill['bill_id'], sponsor))[1:]
            for raw_vote in raw_vote_data:
                raw_vote = raw_vote.split(u'\xa0\xa0\xa0\xa0\xa0\xa0\xa0\xa0\xa0\xa0')
                motion = raw_vote[0]

                vote_date = re.search('(\d+/\d+/\d+)', motion)
                if vote_date:
                    vote_date = datetime.datetime.strptime(vote_date.group(), '%m/%d/%Y') 

                passed = ('Passed' in motion) or ('Adopted' in raw_vote[1])
                vote_regex = re.compile('\d+$')
                aye_regex = re.compile('^.+voting aye were: (.+) -')
                no_regex = re.compile('^.+voting no were: (.+) -')
                yes_count = None
                no_count = None
                other_count = 0
                ayes = []
                nos = []
                
                for v in raw_vote[1:]:
                    if v.startswith('Ayes...') and vote_regex.search(v):
                        yes_count = int(vote_regex.search(v).group())
                    elif v.startswith('Noes...') and vote_regex.search(v):
                        no_count = int(vote_regex.search(v).group())
                    elif aye_regex.search(v):
                        ayes = aye_regex.search(v).groups()[0].split(', ')
                    elif no_regex.search(v):
                        nos = no_regex.search(v).groups()[0].split(', ')

                if yes_count and no_count:
                    passed = yes_count > no_count
                else:
                    yes_count = no_count = 0


                vote = Vote(bill['chamber'], vote_date, motion, passed, yes_count, no_count, other_count) 
                vote.add_source(link)
                for a in ayes:
                    vote.yes(a)
                for n in nos:
                    vote.no(n)
                bill.add_vote(vote)

        return bill

# 需要导入模块: from fiftystates.scrape.votes import Vote [as 别名]
# 或者: from fiftystates.scrape.votes.Vote import add_source [as 别名]
    def scrape_lower_vote(self, url):
        with self.urlopen(url) as page:
            page = lxml.html.fromstring(page)

            table = page.xpath("/html/body/table/tr[3]/td/table/tr/"
                               "td[3]/table/tr/td/table[3]")[0]

            motion = ""
            for part in ("Amendment Number", "Reading Number",
                         "Floor Actions"):
                motion += page.xpath("string(//*[contains(text(), '%s')])" %
                                     part).strip() + " "

            motion = motion.strip()

            date = page.xpath(
                'string(//*[contains(text(), "Date:")]/following-sibling::*)')
            date = datetime.datetime.strptime(date, "%m/%d/%Y")

            yeas = page.xpath('string(//*[contains(text(), "Yeas")])')
            yeas = int(yeas.split(' - ')[1])

            nays = page.xpath('string(//*[contains(text(), "Nays")])')
            nays = int(nays.split(' - ')[1])

            nv = page.xpath('string(//*[contains(text(), "Not Voting")])')
            nv = int(nv.split(' - ')[1])

            passed = yeas > (nays + nv)

            vote = Vote('lower', date, motion, passed, yeas, nays, nv)
            vote.add_source(url)

            for tr in table.xpath("tr/td/table/tr"):
                text = tr.xpath("string()")
                text = re.sub(r"\s+", r" ", text)

                name = " ".join(text.split()[1:])

                if text[0] == "Y":
                    vote.yes(name)
                elif text[0] == "N":
                    vote.no(name)
                elif text[0] in ("-", "C"):
                    vote.other(name)

            return vote

# 需要导入模块: from fiftystates.scrape.votes import Vote [as 别名]
# 或者: from fiftystates.scrape.votes.Vote import add_source [as 别名]
    def scrape_new_vote(self, url):
        vote_page = self.soup_parser(self.urlopen(url))

        header = vote_page.find(id="ctl00_contentMain_hdVote").contents[0]

        chamber_name = header.split(', ')[1]
        if chamber_name.startswith('House'):
            chamber = 'lower'
        else:
            chamber = 'upper'

        location = ' '.join(chamber_name.split(' ')[1:])
        if location.startswith('of Representatives'):
            location = ''

        motion = ', '.join(header.split(', ')[2:])

        yes_count = int(vote_page.find(
            id="ctl00_contentMain_tdAyes").contents[0])
        no_count = int(vote_page.find(
            id="ctl00_contentMain_tdNays").contents[0])
        excused_count = int(vote_page.find(
            id="ctl00_contentMain_tdExcused").contents[0])
        absent_count = int(vote_page.find(
            id="ctl00_contentMain_tdAbsent").contents[0])
        other_count = excused_count + absent_count

        passed = yes_count > no_count

        vote = Vote(chamber, None, motion, passed,
                    yes_count, no_count,
                    other_count, excused_count=excused_count,
                    absent_count=absent_count,
                    location=location)
        vote.add_source(url)

        vote_tbl = vote_page.find(id="ctl00_contentMain_tblVotes")
        for td in vote_tbl.findAll('td'):
            if td.contents[0] == 'Yea':
                vote.yes(td.findPrevious().contents[0])
            elif td.contents[0] == 'Nay':
                vote.no(td.findPrevious().contents[0])
            elif td.contents[0] in ['Excused', 'Absent']:
                vote.other(td.findPrevious().contents[0])

        return vote

# 需要导入模块: from fiftystates.scrape.votes import Vote [as 别名]
# 或者: from fiftystates.scrape.votes.Vote import add_source [as 别名]
    def parse_vote(self, bill, actor, date, motion, url):
        with self.urlopen(url) as page:
            vote_re = re.compile('YEAS -?\s?(\d+)(.*)NAYS -?\s?(\d+)'
                                 '(.*)ABSENT( OR NOT VOTING)? -?\s?'
                                 '(\d+)(.*)',
                                 re.MULTILINE | re.DOTALL)
            match = vote_re.search(page)
            yes_count = int(match.group(1))
            no_count = int(match.group(3))
            other_count = int(match.group(6))

            if yes_count > no_count:
                passed = True
            else:
                passed = False

            if actor == 'upper' or actor == 'lower':
                vote_chamber = actor
                vote_location = ''
            else:
                vote_chamber = ''
                vote_location = actor

            vote = Vote(vote_chamber, date,
                        motion, passed, yes_count, no_count,
                        other_count,
                        location=vote_location)
            vote.add_source(url)

            yes_votes = re.split('\s{2,}', match.group(2).strip())
            no_votes = re.split('\s{2,}', match.group(4).strip())
            other_votes = re.split('\s{2,}', match.group(7).strip())

            for yes in yes_votes:
                if yes:
                    vote.yes(yes)
            for no in no_votes:
                if no:
                    vote.no(no)
            for other in other_votes:
                if other:
                    vote.other(other)

            bill.add_vote(vote)

# 需要导入模块: from fiftystates.scrape.votes import Vote [as 别名]
# 或者: from fiftystates.scrape.votes.Vote import add_source [as 别名]
    def scrape_vote(self, bill, chamber, url):
        with self.urlopen(url) as page:
            page = page.replace(' ', ' ')
            page = lxml.html.fromstring(page)

            info_row = page.xpath("//table[1]/tr[2]")[0]

            date = info_row.xpath("string(td[1])")
            date = datetime.datetime.strptime(date, "%m/%d/%Y")

            motion = info_row.xpath("string(td[2])")
            yes_count = int(info_row.xpath("string(td[3])"))
            no_count = int(info_row.xpath("string(td[4])"))
            other_count = int(info_row.xpath("string(td[5])"))
            passed = info_row.xpath("string(td[6])") == 'Pass'

            if motion == 'Shall the bill pass?':
                type = 'passage'
            elif motion == 'Shall the bill be read the third time?':
                type = 'reading:3'
            elif 'be amended as' in motion:
                type = 'amendment'
            else:
                type = 'other'

            vote = Vote(chamber, date, motion, passed,
                        yes_count, no_count, other_count)
            vote.add_source(url)

            for tr in page.xpath("//table[1]/tr")[3:]:
                if len(tr.xpath("td")) != 2:
                    continue

                name = tr.xpath("string(td[1])").split(' of')[0]

                type = tr.xpath("string(td[2])").strip()
                if type == 'Yea':
                    vote.yes(name)
                elif type == 'Nay':
                    vote.no(name)
                else:
                    vote.other(name)

            bill.add_vote(vote)

# 需要导入模块: from fiftystates.scrape.votes import Vote [as 别名]
# 或者: from fiftystates.scrape.votes.Vote import add_source [as 别名]
 def scrape_votes(self, vote_text, vote_url, house, date, bill):
     votes_parts = vote_text.split(";")
     voters = []
                             
     motion_text, sep, after = vote_text.partition(" The votes were as follows:")
                             
     for vp in votes_parts:
         before, sep, after = vp.partition("(s)")
         voters_list = after.split(", ")
         voters_list[0] = voters_list[0].lstrip(" ")
         voters_list[-1] = voters_list[-1].rstrip(". ")                          
         voters.append(voters_list)
                             
     #Ayes, Ayes with reservations, Noes, Excused
                             
     vote_counts = [0, 0, 0, 0]
                             
     for i, t in enumerate(votes_parts):
         match = re.search("[0-9]+", t)
         if (match != None):
             vote_counts[i] = int(match.group(0))
                             
     if(house == 'H'):
         vote_house = "lower"
     else:
         vote_house = "upper"
                             
     vote = Vote(vote_house, date, motion_text, True, \
             vote_counts[0], vote_counts[2], vote_counts[1] + vote_counts[3])
     vote.add_source(vote_url)
                             
     for yes_voter in voters[0]:
         vote.yes(yes_voter)
     for no_voter in voters[2]:
         vote.no(no_voter)
     for other_voter in voters[1]:
         vote.other(other_voter)
     for other_voter in voters[2]:
         vote.other(other_voter)  
     
     bill.add_vote(vote)