Python LegistarScraper.expandHistorySummary方法代码示例

# 需要导入模块: from legistar.scraper import LegistarScraper [as 别名]
# 或者: from legistar.scraper.LegistarScraper import expandHistorySummary [as 别名]
def can_get_history_detail_using_summary_row():
    config = {"hostname": "phila.legistar.com", "fulltext": True}
    scraper = LegistarScraper(config)
    legislation_summary = {
        "URL": "http://phila.legistar.com/LegislationDetail.aspx?ID=1236768&GUID=EB92A4C2-469A-4D73-97C0-A620BBDDD5BE&Options=ID|Text|&Search="
    }
    legislation_details = scraper.expandLegislationSummary(legislation_summary)
    history_summary = legislation_details[1][2]

    attrs, votes = scraper.expandHistorySummary(history_summary)
    ayes = [vote for vote in votes if vote["Vote"] == "Ayes"]
    assert_equal(len(ayes), 14)
    assert_equal(attrs["Result"], "Pass")

# 需要导入模块: from legistar.scraper import LegistarScraper [as 别名]
# 或者: from legistar.scraper.LegistarScraper import expandHistorySummary [as 别名]
def can_get_history_detail_using_summary_row():
    config = Config(hostname="chicago.legistar.com", sponsor_links=False).defaults(DEFAULT_CONFIG)
    scraper = LegistarScraper(config)
    legislation_summary = {
        "URL": "https://chicago.legistar.com/LegislationDetail.aspx?ID=1450228&GUID=97689689-D0EA-47A2-8474-09B3A71C221B&Options=Advanced&Search="
    }
    legislation_details = scraper.expandLegislationSummary(legislation_summary)

    history_summary = legislation_details[1][0]["Action Details"]

    attrs, votes = scraper.expandHistorySummary(history_summary)
    ayes = [vote for vote in votes if vote["Vote"] == "Yea"]
    assert_equal(len(ayes), 49)
    assert_equal(attrs["Result"], "Pass")

# 需要导入模块: from legistar.scraper import LegistarScraper [as 别名]
# 或者: from legistar.scraper.LegistarScraper import expandHistorySummary [as 别名]
class HostedLegistarSiteWrapper (object):
    """
    A generic facade over hosted legistar site data scraper.
    It is responsible for interpreting data scraped out of the site by LegistarScraper.
    The main external point of interaction is scrape_legis_file.
    
    NOTE that this is a superclass that will not run by itself and isn't
    meant to be; you are expected to run a subclass that implements
    some functions with names starting with "pluck".

    requires: BeautifulSoup, mechanize
    """

    def __init__(self, cmdline_options, **options):
        self.scraper = LegistarScraper(options)
        if cmdline_options['year']:
          self.legislation_summaries =  self.scraper.searchLegislation('',year=cmdline_options['year'])
        else:
          self.legislation_summaries =  self.scraper.searchLegislation('')

    def scrape_legis_file(self, key, summary):
        '''Extract a record from the given document (soup). The key is for the
           sake of record-keeping.  It is the key passed to the site URL.'''

        while True :
            try:
                legislation_attrs, legislation_history = self.scraper.expandLegislationSummary(summary)
                break
            except urllib2.URLError as e:
                print e
                print 'skipping to next leg record'
            except AttributeError as e :
                print e
                print 'skipping to next leg record'
            while True :
                try:
                    summary = self.legislation_summaries.next()
                    break
                except urllib2.URLError as e:
                    print e
                    print 'sleeping for five minutes'
                    time.sleep('360')

        parsed_url = urlparse.urlparse(summary['URL'])
        key = urlparse.parse_qs(parsed_url.query)['ID'][0]
        
        record = self.pluck_record(key, summary, legislation_attrs)

        attachments = self.pluck_attachments(key, legislation_attrs)

        actions = []
        for act in legislation_history :
            act_details = act_votes = []
            try:
                act_details, act_votes = self.scraper.expandHistorySummary(act)
            except (KeyError, AttributeError) as e:
                log.debug('LegAction has no url')
            else:
              if act_votes:
                print "act_votes", act_votes
              
            try:
                action = self.pluck_action(key, act, act_details, act_votes)
            except TypeError as e:
                print e
                print summary
                continue
            except KeyError as e :
                print act
                print e
                print summary
                raise
            actions.append(action)

        # we should probably remove this from the model since the hosted
        # legistar does not have minutes
        minutes = []

        log.info('Scraped legfile with key %r' % (key,))
        log.debug("%r %r %r %r" % (record, attachments, actions, minutes))

        return record, attachments, actions, minutes

    def convert_date(self, orig_date):
        if orig_date:
            return datetime.datetime.strptime(orig_date, '%m/%d/%Y').date()
        else:
            return ''


    def check_for_new_content(self, last_key):
        '''Grab the next legislation summary row. Doesn't use the last_key
           parameter; just starts at the beginning for each instance of the
           scraper.
        '''
        try:
            print 'next leg record'
            next_summary = self.legislation_summaries.next()
            return 0, next_summary
        except StopIteration:
#.........这里部分代码省略.........

# 需要导入模块: from legistar.scraper import LegistarScraper [as 别名]
# 或者: from legistar.scraper.LegistarScraper import expandHistorySummary [as 别名]
class HostedLegistarSiteWrapper (object):
    """
    A facade over the Philadelphia city council legistar site data.  It is
    responsible for scraping data out of the site.  The main external point
    of interaction is scrape_legis_file.

    requires: BeautifulSoup, mechanize
    """

    def __init__(self, **options):
        self.scraper = LegistarScraper(options)
        self.legislation_summaries =  self.scraper.searchLegislation('', created_before='2012-10-5')

    def scrape_legis_file(self, key, summary):
        '''Extract a record from the given document (soup). The key is for the
           sake of record-keeping.  It is the key passed to the site URL.'''

        while True :
            try:
                legislation_attrs, legislation_history = self.scraper.expandLegislationSummary(summary)
                break
            except urllib2.URLError as e:
                print e
                print 'skipping to next leg record'
            except AttributeError as e :
                print e
                print 'skipping to next leg record'
            while True :
                try:
                    summary = self.legislation_summaries.next()
                    break
                except urllib2.URLError as e:
                    print e
                    print 'sleeping for five minutes'
                    time.sleep('360')



            
        parsed_url = urlparse.urlparse(summary['URL'])
        key = urlparse.parse_qs(parsed_url.query)['ID'][0]
        
        # re-order the sponsor name by '[First] [Last]' instead of '[Last], [First]'
        sponsors = legislation_attrs['Sponsors']
        first_name_first_sponsors = []
        for sponsor in sponsors :
            if ',' in sponsor :
                name_list = sponsor.rsplit(',', 1)
                name_list.reverse()
                sponsor = ' '.join(name_list).strip().replace(',', '')
            first_name_first_sponsors.append(sponsor)

        record = {
            'key' : key,
            'id' : summary['Record #'],
            'url' : summary['URL'],
            'type' : summary['Type'],
            'status' : summary['Status'],
            'title' : summary['Title'],
            'controlling_body' : legislation_attrs['Current Controlling Legislative Body'],
            'intro_date' : self.convert_date(summary['Intro Date']),
            'final_date' : self.convert_date(summary.setdefault('Final Date', '')),
            'version' : summary.setdefault('Version', ''),
            #'contact' : None,
            'sponsors' : first_name_first_sponsors,
            # probably remove this from the model as well
            'minutes_url'  : None
        }

        try:
            attachments = legislation_attrs['Attachments']
            for attachment in attachments:
                attachment['key'] = key
                attachment['file'] = attachment['label']
                attachment['description'] = attachment['label']
                del attachment['label']
        except KeyError:
            attachments = []

        actions = []
        for act in legislation_history :
            try:
                act_details, act_votes = self.scraper.expandHistorySummary(act)
            except (KeyError, AttributeError) as e:
                print e
                print summary
                continue
            try:
                action = {
                    'key' : key,
                    'date_taken' : self.convert_date(act['Date']),
                    'acting_body' : act['Action By']['label'],
                    'motion' : act['Result'],
                    'description' : act['Action'],
                    'notes' : ''
                    }
            except TypeError as e:
                print e
                print summary
                continue
#.........这里部分代码省略.........