Python LegistarScraper.expandLegislationSummary方法代码示例

# 需要导入模块: from legistar.scraper import LegistarScraper [as 别名]
# 或者: from legistar.scraper.LegistarScraper import expandLegislationSummary [as 别名]
def philly_topics():
    """Tests that scraper works for Philly legislation with and without topics"""
    config = Config(hostname="philly.legistar.com").defaults(DEFAULT_CONFIG)
    scraper = LegistarScraper(config)
    legislation_with_topics = {
        "URL": "http://phila.legistar.com/LegislationDetail.aspx?ID=1433307&GUID=773A9C3F-ABA5-4D6C-B901-A9EEE3B1B8B0"
    }
    legislation_details = scraper.expandLegislationSummary(legislation_with_topics)
    assert_equal(legislation_details[0]["Topics"], [u"LIQUOR BY THE DRINK TAX", u"SCHOOL TAX AUTHORIZATION"])
    legislation_no_topics = {
        "URL": "http://phila.legistar.com/LegislationDetail.aspx?ID=1426307&GUID=E9EC8885-0DDD-4B64-AB2D-EA0503284268"
    }
    legislation_details = scraper.expandLegislationSummary(legislation_no_topics)
    assert_equal(legislation_details[0]["Topics"], [])

# 需要导入模块: from legistar.scraper import LegistarScraper [as 别名]
# 或者: from legistar.scraper.LegistarScraper import expandLegislationSummary [as 别名]
def philly_sponsors():
    config = Config(hostname="phila.legistar.com", sponsor_links=False).defaults(DEFAULT_CONFIG)
    scraper = LegistarScraper(config)
    legislation_summary = {
        "URL": "http://phila.legistar.com/LegislationDetail.aspx?ID=1233260&GUID=DC103FB6-FF9D-4250-B0CE-111B80E8B80C"
    }
    legislation_details = scraper.expandLegislationSummary(legislation_summary)
    assert_equal(legislation_details[0]["Sponsors"][0], u"Councilmember DiCicco")

# 需要导入模块: from legistar.scraper import LegistarScraper [as 别名]
# 或者: from legistar.scraper.LegistarScraper import expandLegislationSummary [as 别名]
def chicago_topics():
    """Tests that scraper works for Chicago for legislation with and without topics"""
    config = Config(hostname="chicago.legistar.com").defaults(DEFAULT_CONFIG)
    scraper = LegistarScraper(config)
    legislation_with_topics = {
        "URL": "http://chicago.legistar.com/LegislationDetail.aspx?ID=1319481&GUID=40B01792-C9D8-4E8C-BADE-2D27BFC8284D"
    }
    legislation_details = scraper.expandLegislationSummary(legislation_with_topics)

    print legislation_details[0]
    assert_equal(legislation_details[0]["Topics"], [u"PUBLIC WAY USAGE - Awnings"])

    legislation_no_topics = {
        "URL": "http://chicago.legistar.com/LegislationDetail.aspx?ID=1429779&GUID=118DDF75-D698-4526-BA54-B560BB6CCB04"
    }
    legislation_details = scraper.expandLegislationSummary(legislation_no_topics)
    assert_equal(legislation_details[0]["Topics"], [])

# 需要导入模块: from legistar.scraper import LegistarScraper [as 别名]
# 或者: from legistar.scraper.LegistarScraper import expandLegislationSummary [as 别名]
def parse_sponsors():
    config = {"hostname": "chicago.legistar.com", "fulltext": True}
    scraper = LegistarScraper(config)
    legislation_summary = {
        "URL": "http://chicago.legistar.com/LegislationDetail.aspx?ID=1255978&GUID=8051C1E6-DED6-433B-AC9A-0FE436051C9F"
    }
    legislation_details = scraper.expandLegislationSummary(legislation_summary)
    assert_equal(legislation_details[0]["Sponsors"][1], u"Moreno, Proco Joe")

# 需要导入模块: from legistar.scraper import LegistarScraper [as 别名]
# 或者: from legistar.scraper.LegistarScraper import expandLegislationSummary [as 别名]
def parse_detail_keys():
    config = Config(hostname="phila.legistar.com", fulltext=True).defaults(DEFAULT_CONFIG)
    scraper = LegistarScraper(config)
    summary = {
        "URL": "http://phila.legistar.com/LegislationDetail.aspx?ID=1265815&GUID=97CBBF7C-A123-4808-9D50-A1E340BE5BC1"
    }
    detail = scraper.expandLegislationSummary(summary)
    assert_in(u"Version", detail[0].keys())
    assert_not_in(u"CITY COUNCIL", detail[0].keys())

# 需要导入模块: from legistar.scraper import LegistarScraper [as 别名]
# 或者: from legistar.scraper.LegistarScraper import expandLegislationSummary [as 别名]
def attachments_list():
    config = Config(hostname="phila.legistar.com", sponsor_links=False).defaults(DEFAULT_CONFIG)
    scraper = LegistarScraper(config)
    detail = scraper.expandLegislationSummary(
        {
            "URL": "http://phila.legistar.com/LegislationDetail.aspx?ID=1243262&GUID=01021C5A-3624-4E5D-AA32-9822D1F5DA29&Options=ID|Text|&Search="
        }
    )
    # Attachments value should be a list
    assert_is_instance(detail[0]["Attachments"], list)

# 需要导入模块: from legistar.scraper import LegistarScraper [as 别名]
# 或者: from legistar.scraper.LegistarScraper import expandLegislationSummary [as 别名]
def attachments_list():
    config = {"hostname": "phila.legistar.com", "fulltext": True}
    scraper = LegistarScraper(config)
    detail = scraper.expandLegislationSummary(
        {
            "URL": "http://phila.legistar.com/LegislationDetail.aspx?ID=1243262&GUID=01021C5A-3624-4E5D-AA32-9822D1F5DA29&Options=ID|Text|&Search="
        }
    )
    # Attachments value should be a list
    assert_is_instance(detail[0]["Attachments"], list)

# 需要导入模块: from legistar.scraper import LegistarScraper [as 别名]
# 或者: from legistar.scraper.LegistarScraper import expandLegislationSummary [as 别名]
def no_attachments_list():
    config = {"hostname": "phila.legistar.com", "fulltext": True}
    scraper = LegistarScraper(config)
    detail = scraper.expandLegislationSummary(
        {
            "URL": "http://phila.legistar.com/LegislationDetail.aspx?ID=1254964&GUID=AF8A4E91-4DF6-41A2-80B4-EFC94A2AFF89&Options=ID|Text|&Search="
        }
    )
    # Legislation with no attachments should have no attachment key
    assert_not_in("Attachments", detail[0])

# 需要导入模块: from legistar.scraper import LegistarScraper [as 别名]
# 或者: from legistar.scraper.LegistarScraper import expandLegislationSummary [as 别名]
def no_attachments_list():
    config = Config(hostname="phila.legistar.com", sponsor_links=False).defaults(DEFAULT_CONFIG)

    scraper = LegistarScraper(config)
    detail = scraper.expandLegislationSummary(
        {
            "URL": "http://phila.legistar.com/LegislationDetail.aspx?ID=1254964&GUID=AF8A4E91-4DF6-41A2-80B4-EFC94A2AFF89&Options=ID|Text|&Search="
        }
    )
    # Legislation with no attachments should have no attachment key
    assert_not_in("Attachments", detail[0])

# 需要导入模块: from legistar.scraper import LegistarScraper [as 别名]
# 或者: from legistar.scraper.LegistarScraper import expandLegislationSummary [as 别名]
def history_row_url():
    config = Config(hostname="chicago.legistar.com", fulltext=True).defaults(DEFAULT_CONFIG)
    scraper = LegistarScraper(config)
    detail = scraper.expandLegislationSummary(
        {
            "URL": "http://chicago.legistar.com/LegislationDetail.aspx?ID=1255978&GUID=8051C1E6-DED6-433B-AC9A-0FE436051C9F&Options=Advanced&Search="
        }
    )
    assert_equal(
        detail[1][0]["Action Details"]["url"],
        "https://chicago.legistar.com/HistoryDetail.aspx?ID=6534991&GUID=253AA818-B592-4594-8237-0A617AA41766",
    )

# 需要导入模块: from legistar.scraper import LegistarScraper [as 别名]
# 或者: from legistar.scraper.LegistarScraper import expandLegislationSummary [as 别名]
def history_row_url():
    config = {"hostname": "chicago.legistar.com", "fulltext": True}
    scraper = LegistarScraper(config)
    detail = scraper.expandLegislationSummary(
        {
            "URL": "http://chicago.legistar.com/LegislationDetail.aspx?ID=1255978&GUID=8051C1E6-DED6-433B-AC9A-0FE436051C9F&Options=Advanced&Search="
        }
    )
    assert_equal(
        detail[1][0]["URL"],
        "http://chicago.legistar.com/HistoryDetail.aspx?ID=6534991&GUID=253AA818-B592-4594-8237-0A617AA41766",
    )

# 需要导入模块: from legistar.scraper import LegistarScraper [as 别名]
# 或者: from legistar.scraper.LegistarScraper import expandLegislationSummary [as 别名]
def can_get_history_detail_using_summary_row():
    config = {"hostname": "phila.legistar.com", "fulltext": True}
    scraper = LegistarScraper(config)
    legislation_summary = {
        "URL": "http://phila.legistar.com/LegislationDetail.aspx?ID=1236768&GUID=EB92A4C2-469A-4D73-97C0-A620BBDDD5BE&Options=ID|Text|&Search="
    }
    legislation_details = scraper.expandLegislationSummary(legislation_summary)
    history_summary = legislation_details[1][2]

    attrs, votes = scraper.expandHistorySummary(history_summary)
    ayes = [vote for vote in votes if vote["Vote"] == "Ayes"]
    assert_equal(len(ayes), 14)
    assert_equal(attrs["Result"], "Pass")

# 需要导入模块: from legistar.scraper import LegistarScraper [as 别名]
# 或者: from legistar.scraper.LegistarScraper import expandLegislationSummary [as 别名]
def can_get_history_detail_using_summary_row():
    config = Config(hostname="chicago.legistar.com", sponsor_links=False).defaults(DEFAULT_CONFIG)
    scraper = LegistarScraper(config)
    legislation_summary = {
        "URL": "https://chicago.legistar.com/LegislationDetail.aspx?ID=1450228&GUID=97689689-D0EA-47A2-8474-09B3A71C221B&Options=Advanced&Search="
    }
    legislation_details = scraper.expandLegislationSummary(legislation_summary)

    history_summary = legislation_details[1][0]["Action Details"]

    attrs, votes = scraper.expandHistorySummary(history_summary)
    ayes = [vote for vote in votes if vote["Vote"] == "Yea"]
    assert_equal(len(ayes), 49)
    assert_equal(attrs["Result"], "Pass")

# 需要导入模块: from legistar.scraper import LegistarScraper [as 别名]
# 或者: from legistar.scraper.LegistarScraper import expandLegislationSummary [as 别名]
class HostedLegistarSiteWrapper (object):
    """
    A generic facade over hosted legistar site data scraper.
    It is responsible for interpreting data scraped out of the site by LegistarScraper.
    The main external point of interaction is scrape_legis_file.
    
    NOTE that this is a superclass that will not run by itself and isn't
    meant to be; you are expected to run a subclass that implements
    some functions with names starting with "pluck".

    requires: BeautifulSoup, mechanize
    """

    def __init__(self, cmdline_options, **options):
        self.scraper = LegistarScraper(options)
        if cmdline_options['year']:
          self.legislation_summaries =  self.scraper.searchLegislation('',year=cmdline_options['year'])
        else:
          self.legislation_summaries =  self.scraper.searchLegislation('')

    def scrape_legis_file(self, key, summary):
        '''Extract a record from the given document (soup). The key is for the
           sake of record-keeping.  It is the key passed to the site URL.'''

        while True :
            try:
                legislation_attrs, legislation_history = self.scraper.expandLegislationSummary(summary)
                break
            except urllib2.URLError as e:
                print e
                print 'skipping to next leg record'
            except AttributeError as e :
                print e
                print 'skipping to next leg record'
            while True :
                try:
                    summary = self.legislation_summaries.next()
                    break
                except urllib2.URLError as e:
                    print e
                    print 'sleeping for five minutes'
                    time.sleep('360')

        parsed_url = urlparse.urlparse(summary['URL'])
        key = urlparse.parse_qs(parsed_url.query)['ID'][0]
        
        record = self.pluck_record(key, summary, legislation_attrs)

        attachments = self.pluck_attachments(key, legislation_attrs)

        actions = []
        for act in legislation_history :
            act_details = act_votes = []
            try:
                act_details, act_votes = self.scraper.expandHistorySummary(act)
            except (KeyError, AttributeError) as e:
                log.debug('LegAction has no url')
            else:
              if act_votes:
                print "act_votes", act_votes
              
            try:
                action = self.pluck_action(key, act, act_details, act_votes)
            except TypeError as e:
                print e
                print summary
                continue
            except KeyError as e :
                print act
                print e
                print summary
                raise
            actions.append(action)

        # we should probably remove this from the model since the hosted
        # legistar does not have minutes
        minutes = []

        log.info('Scraped legfile with key %r' % (key,))
        log.debug("%r %r %r %r" % (record, attachments, actions, minutes))

        return record, attachments, actions, minutes

    def convert_date(self, orig_date):
        if orig_date:
            return datetime.datetime.strptime(orig_date, '%m/%d/%Y').date()
        else:
            return ''


    def check_for_new_content(self, last_key):
        '''Grab the next legislation summary row. Doesn't use the last_key
           parameter; just starts at the beginning for each instance of the
           scraper.
        '''
        try:
            print 'next leg record'
            next_summary = self.legislation_summaries.next()
            return 0, next_summary
        except StopIteration:
#.........这里部分代码省略.........

# 需要导入模块: from legistar.scraper import LegistarScraper [as 别名]
# 或者: from legistar.scraper.LegistarScraper import expandLegislationSummary [as 别名]
class HostedLegistarSiteWrapper (object):
    """
    A facade over the Philadelphia city council legistar site data.  It is
    responsible for scraping data out of the site.  The main external point
    of interaction is scrape_legis_file.

    requires: BeautifulSoup, mechanize
    """

    def __init__(self, **options):
        self.id_label = options.pop('id_label', 'Record #')
        self.url_label = options.pop('url_label', 'URL')
        self.type_label = options.pop('type_label', 'Type')
        self.status_label = options.pop('status_label', 'Status')
        self.title_label = options.pop('title_label', 'Title')
        self.indexes_label = options.pop('indexes_label', 'Indexes')
        self.intro_date_label = options.pop('intro_date_label', 'Intro Date')
        self.final_date_label = options.pop('final_date_label', 'Final Date')
        self.controlling_body_label = options.pop('controlling_body_label', 'Current Controlling Legislative Body')
        self.version_label = options.pop('version_label', 'Version')

        self.scraper = LegistarScraper(options)
        self.legislation_summaries =  self.scraper.searchLegislation('', created_before='2012-10-5')

    def scrape_legis_file(self, key, summary):
        '''Extract a record from the given document (soup). The key is for the
           sake of record-keeping.  It is the key passed to the site URL.'''

        while True :
            try:
                legislation_attrs, legislation_history = self.scraper.expandLegislationSummary(summary)
                break
            except urllib2.URLError as e:
                log.warning(e)
                log.warning('skipping to next leg record')
            except AttributeError as e :
                log.warning(e)
                log.warning('skipping to next leg record')
            while True :
                try:
                    summary = self.legislation_summaries.next()
                    break
                except urllib2.URLError as e:
                    log.warning(e)
                    log.warning('sleeping for five minutes')
                    time.sleep('360')



            
        parsed_url = urlparse.urlparse(summary['URL'])
        key = urlparse.parse_qs(parsed_url.query)['ID'][0]
        
        # re-order the sponsor name by '[First] [Last]' instead of '[Last], [First]'
        sponsors = legislation_attrs['Sponsors']
        first_name_first_sponsors = []
        for sponsor in sponsors :
            if ',' in sponsor :
                name_list = sponsor.split(',')
                name_list.reverse()
                sponsor = ' '.join(name_list).strip()
            first_name_first_sponsors.append(sponsor)

        try:
            record = {
                'key' : key,
                'id' : summary[self.id_label],
                'url' : summary[self.url_label],
                'type' : summary[self.type_label],
                'status' : summary[self.status_label],
                'title' : summary[self.title_label],
                'indexes': legislation_attrs[self.indexes_label],
                'controlling_body' : legislation_attrs[self.controlling_body_label],
                'intro_date' : self.convert_date(summary[self.intro_date_label]),
                'final_date' : self.convert_date(summary.setdefault(self.final_date_label, '')),
                'version' : summary.setdefault(self.version_label, ''),
                #'contact' : None,
                'sponsors' : first_name_first_sponsors,
                # probably remove this from the model as well
                'minutes_url'  : None
            }
        except KeyError, e:
            raise ScrapeError('Failed to find key %s in either summary keys '
                              '(%r) or attrs (%r)' % (e, summary.keys(), 
                                                    legislation_attrs.keys()))

        try:
            attachments = legislation_attrs['Attachments']
            for attachment in attachments:
                attachment['key'] = key
                attachment['file'] = attachment['label']
                attachment['description'] = attachment['label']
                del attachment['label']
        except KeyError:
            attachments = []

        actions = []
        for act in legislation_history :
            try:
                act_details, act_votes = self.scraper.expandHistorySummary(act)
#.........这里部分代码省略.........