Python httpretty.register_uri函数代码示例

 def testMainCreatesCollectionProfile(self, mock_boto3):
     '''Test that the main function produces a collection profile
     file for DPLA. The path to this file is needed when creating a
     DPLA ingestion document.
     '''
     httpretty.register_uri(
         httpretty.GET,
         "https://registry.cdlib.org/api/v1/collection/197/",
         body=open(DIR_FIXTURES + '/collection_api_test.json').read())
     httpretty.register_uri(
         httpretty.GET,
         re.compile("http://content.cdlib.org/oai?.*"),
         body=open(DIR_FIXTURES + '/testOAI-128-records.xml').read())
     Collection("https://registry.cdlib.org/api/v1/collection/197/")
     with patch('dplaingestion.couch.Couch') as mock_couch:
         instance = mock_couch.return_value
         instance._create_ingestion_document.return_value = 'test-id'
         ingest_doc_id, num, self.dir_save, self.fetcher = fetcher.main(
             self.user_email,
             self.url_api_collection,
             log_handler=self.test_log_handler,
             mail_handler=self.test_log_handler,
             dir_profile=self.dir_test_profile,
             profile_path=self.profile_path,
             config_file=self.config_file)
     self.assertEqual(ingest_doc_id, 'test-id')
     self.assertEqual(num, 128)
     self.assertTrue(os.path.exists(os.path.join(self.profile_path)))

 def testRunIngestProductionNotReady(self, mock_couch, mock_dash_clean,
                                     mock_check, mock_remove, mock_save,
                                     mock_enrich, mock_couchdb, mock_redis,
                                     mock_boto3):
     mock_couch.return_value._create_ingestion_document.return_value = \
         'test-id'
     # this next is because the redis client unpickles....
     mock_redis.return_value.hget.return_value = pickle.dumps('RQ-result!')
     mail_handler = MagicMock()
     url_api_collection = 'https://registry.cdlib.org/api/v1/' \
         'collection/178/'
     httpretty.httpretty.enable()
     httpretty.register_uri(
         httpretty.GET,
         url_api_collection,
         body=open(DIR_FIXTURES + '/collection_api_test_oac.json').read())
     httpretty.register_uri(
         httpretty.GET,
         'http://dsc.cdlib.org/search?facet=type-tab&style=cui&raw=1&'
         'relation=ark:/13030/tf2v19n928',
         body=open(DIR_FIXTURES + '/testOAC-url_next-1.json').read())
     os.environ['DATA_BRANCH'] = 'production'
     self.assertRaises(
         Exception,
         run_ingest.main,
         '[email protected]',
         url_api_collection,
         log_handler=self.test_log_handler,
         mail_handler=mail_handler)
     print self.test_log_handler.records
     self.assertEqual(len(self.test_log_handler.records), 9)

 def testFetch(self):
     '''Test the httpretty mocked fetching of documents'''
     url = 'https://example.edu/action/search/xml?q=ddu%3A20*&' \
           'asf=ddu&asd=&fd=1&_hd=&hd=on&sf=&_rs=&_ef=&ef=on&sd=&ed=&c=ga'
     httpretty.register_uri(
             httpretty.GET,
             url,
             responses=[
                 httpretty.Response(
                     open(DIR_FIXTURES+'/ucsf-page-1.xml').read(),
                     status=200),
                 httpretty.Response(
                     open(DIR_FIXTURES+'/ucsf-page-1.xml').read(),
                     status=200),
                 httpretty.Response(
                     open(DIR_FIXTURES+'/ucsf-page-2.xml').read(),
                     status=200),
                 httpretty.Response(
                     open(DIR_FIXTURES+'/ucsf-page-3.xml').read(),
                     status=200),
             ]
     )
     h = fetcher.UCSF_XML_Fetcher(url, None, page_size=3)
     docs = []
     for d in h:
         docs.extend(d)
     self.assertEqual(len(docs), 7)
     testy = docs[0]
     self.assertIn('tid', testy)
     self.assertEqual(testy['tid'], "nga13j00")
     self.assertEqual(testy['uri'],
                      'http://legacy.library.ucsf.edu/tid/nga13j00')
     self.assertIn('aup', testy['metadata'])
     self.assertEqual(testy['metadata']['aup'], ['Whent, Peter'])

 def testMainFnWithException(self, mock_method):
     httpretty.register_uri(
         httpretty.GET,
         "https://registry.cdlib.org/api/v1/collection/197/",
         body=open(DIR_FIXTURES + '/collection_api_test.json').read())
     httpretty.register_uri(
         httpretty.GET,
         re.compile("http://content.cdlib.org/oai?.*"),
         body=open(DIR_FIXTURES + '/testOAI-128-records.xml').read())
     with patch('dplaingestion.couch.Couch') as mock_couch:
         instance = mock_couch.return_value
         instance._create_ingestion_document.return_value = 'test-id'
         self.assertRaises(
             Exception,
             fetcher.main,
             self.user_email,
             self.url_api_collection,
             log_handler=self.test_log_handler,
             mail_handler=self.test_log_handler,
             profile_path=self.profile_path,
             config_file=self.config_file)
     self.assertEqual(len(self.test_log_handler.records), 7)
     self.assertTrue("[ERROR] HarvestMain: Error while harvesting:" in
                     self.test_log_handler.formatted_records[6])
     self.assertTrue("Boom!" in self.test_log_handler.formatted_records[6])

 def testInit(self):
     '''Basic tdd start'''
     url = 'https://example.edu'
     user_id = 'testuser'
     page_size = 10
     url_first = fetcher.Flickr_Fetcher.url_get_photos_template.format(
         api_key='boguskey', user_id=user_id, per_page=page_size, page=1)
     httpretty.register_uri(
         httpretty.GET,
         url_first,
         body=open(DIR_FIXTURES + '/flickr-public-photos-1.xml').read())
     h = fetcher.Flickr_Fetcher(url, user_id, page_size=page_size)
     self.assertEqual(h.url_base, url)
     self.assertEqual(h.user_id, user_id)
     self.assertEqual(h.page_size, 10)
     self.assertEqual(h.page_current, 1)
     self.assertEqual(h.doc_current, 0)
     self.assertEqual(h.docs_fetched, 0)
     self.assertEqual(h.url_get_photos_template,
                      'https://api.flickr.com/services/rest/'
                      '?api_key={api_key}&user_id={user_id}&per_page'
                      '={per_page}&method='
                      'flickr.people.getPublicPhotos&page={page}')
     self.assertEqual(h.api_key, 'boguskey')
     self.assertEqual(h.url_current, url_first)
     self.assertEqual(h.docs_total, 10)
     self.assertEqual(h.url_get_photo_info_template,
                      'https://api.flickr.com/services/rest/'
                      '?api_key={api_key}&method='
                      'flickr.photos.getInfo&photo_id={photo_id}')

    def setUp(self):
        super(HarvestControllerTestCase, self).setUp()
        httpretty.register_uri(
            httpretty.GET,
            "https://registry.cdlib.org/api/v1/collection/197/",
            body=open(DIR_FIXTURES + '/collection_api_test.json').read())
        httpretty.register_uri(
            httpretty.GET,
            re.compile("http://content.cdlib.org/oai?.*"),
            body=open(DIR_FIXTURES + '/testOAI-128-records.xml').read())
        self.collection = Collection(
            'https://registry.cdlib.org/api/v1/collection/197/')
        config_file, profile_path = self.setUp_config(self.collection)
        self.controller_oai = fetcher.HarvestController(
            '[email protected]',
            self.collection,
            profile_path=profile_path,
            config_file=config_file)
        self.objset_test_doc = json.load(
            open(DIR_FIXTURES + '/objset_test_doc.json'))

        class myNow(datetime.datetime):
            @classmethod
            def now(cls):
                return cls(2017, 7, 14, 12, 1)

        self.old_dt = datetime.datetime
        datetime.datetime = myNow

 def testMainHarvestController__init__Error(self, mock_method):
     '''Test the try-except block in main when HarvestController not created
     correctly'''
     httpretty.register_uri(
         httpretty.GET,
         "https://registry.cdlib.org/api/v1/collection/197/",
         body=open(DIR_FIXTURES + '/collection_api_test.json').read())
     httpretty.register_uri(
         httpretty.GET,
         re.compile("http://content.cdlib.org/oai?.*"),
         body=open(DIR_FIXTURES + '/testOAI-128-records.xml').read())
     sys.argv = [
         'thisexe', '[email protected]',
         'https://registry.cdlib.org/api/v1/collection/197/'
     ]
     self.assertRaises(
         Exception,
         fetcher.main,
         self.user_email,
         self.url_api_collection,
         log_handler=self.test_log_handler,
         mail_handler=self.test_log_handler,
         dir_profile=self.dir_test_profile)
     self.assertEqual(len(self.test_log_handler.records), 4)
     self.assertTrue("[ERROR] HarvestMain: Exception in harvester init" in
                     self.test_log_handler.formatted_records[3])
     self.assertTrue("Boom!" in self.test_log_handler.formatted_records[3])
     c = Collection('https://registry.cdlib.org/api/v1/collection/197/')
     os.remove(
         os.path.abspath(
             os.path.join(self.dir_test_profile, c.id + '.pjs')))

    def testInit(self):
        '''Basic tdd start'''
        url = 'https://s3.amazonaws.com/pastperfectonline/xmlfiles/museum_231'
        httpretty.register_uri(
            httpretty.GET,
            url,
            body=open(DIR_FIXTURES + '/xml-fetch.xml').read())
        h = fetcher.XML_Fetcher(url, None)
        self.assertEqual(h.url_base, url)
        docs = []
        d = h.next()
        self.assertEqual(len(d), 999)
        docs.extend(d)
        for d in h:
            docs.extend(d)
        self.assertEqual(len(docs), 2320)
        test1 = docs[0]
        test2 = docs[2]
        self.assertIn('title', test1['metadata'])
        self.assertEqual(test1['metadata']['title'], [
            'California desperadoes : stories of early California outlaws in their own word'
        ])

        # test that attributes are captured, even from empty elements
        self.assertEqual(test1['metadata']['q'], ['taken'])
        self.assertEqual(test1['metadata']['d'], ['Kodak'])
        self.assertEqual(test2['metadata']['q'], ['scanned'])
        self.assertEqual(test2['metadata']['d'], ['Epson'])

 def testAddRegistryData(self):
     '''Unittest the _add_registry_data function'''
     httpretty.register_uri(
         httpretty.GET,
         "https://registry.cdlib.org/api/v1/collection/197/",
         body=open(DIR_FIXTURES + '/collection_api_test.json').read())
     httpretty.register_uri(
         httpretty.GET,
         re.compile("http://content.cdlib.org/oai?.*"),
         body=open(DIR_FIXTURES + '/testOAI-128-records.xml').read())
     collection = Collection(
         'https://registry.cdlib.org/api/v1/collection/197/')
     self.tearDown_config()  # remove ones setup in setUp
     self.setUp_config(collection)
     controller = fetcher.HarvestController(
         '[email protected]',
         collection,
         config_file=self.config_file,
         profile_path=self.profile_path)
     obj = {'id': 'fakey', 'otherdata': 'test'}
     self.assertNotIn('collection', obj)
     controller._add_registry_data(obj)
     self.assertIn('collection', obj)
     self.assertEqual(obj['collection'][0]['@id'],
                      'https://registry.cdlib.org/api/v1/collection/197/')
     self.assertNotIn('campus', obj)
     self.assertIn('campus', obj['collection'][0])
     self.assertNotIn('repository', obj)
     self.assertIn('repository', obj['collection'][0])
     # need to test one without campus
     self.assertEqual(obj['collection'][0]['campus'][0]['@id'],
                      'https://registry.cdlib.org/api/v1/campus/12/')
     self.assertEqual(obj['collection'][0]['repository'][0]['@id'],
                      'https://registry.cdlib.org/api/v1/repository/37/')

    def test_get_isShownBy_pdf(self, mock_deepharvest, mock_boto):
        ''' test getting correct isShownBy value for Nuxeo doc
            with no images and PDF at parent level
        '''
        deepharvest_mocker(mock_deepharvest)

        httpretty.register_uri(
            httpretty.GET,
            'https://example.edu/api/v1/path/@search?query=SELECT+%2A+FROM+'
            'Document+WHERE+ecm%3AparentId+%3D+'
            '%2700d55837-01b6-4211-80d8-b966a15c257e%27+ORDER+BY+'
            'ecm%3Apos&currentPageIndex=0&pageSize=100',
            responses=[
                httpretty.Response(
                    body=open(DIR_FIXTURES + '/nuxeo_no_children.json').read(),
                    status=200),
            ])

        h = fetcher.NuxeoFetcher('https://example.edu/api/v1',
                                 'path-to-asset/here')

        nuxeo_metadata = open(DIR_FIXTURES +
                              '/nuxeo_doc_pdf_parent.json').read()
        nuxeo_metadata = json.loads(nuxeo_metadata)
        isShownBy = h._get_isShownBy(nuxeo_metadata)
        self.assertEqual(
            isShownBy, 'https://s3.amazonaws.com/static.ucldc.cdlib.org/'
            'ucldc-nuxeo-thumb-media/00d55837-01b6-4211-80d8-b966a15c257e')

    def test_get_isShownBy_video(self, mock_deepharvest, mock_boto):
        ''' test getting correct isShownBy value for Nuxeo video object
        '''
        deepharvest_mocker(mock_deepharvest)

        httpretty.register_uri(
            httpretty.GET,
            'https://example.edu/api/v1/path/@search?query=SELECT+%2A+FROM+'
            'Document+WHERE+ecm%3AparentId+%3D+'
            '%274c80e254-6def-4230-9f28-bc48878568d4%27+'
            'AND+ecm%3AcurrentLifeCycleState+%21%3D+%27deleted%27+ORDER+BY+'
            'ecm%3Apos&currentPageIndex=0&pageSize=100',
            responses=[
                httpretty.Response(
                    body=open(DIR_FIXTURES + '/nuxeo_no_children.json').read(),
                    status=200),
            ])

        h = fetcher.NuxeoFetcher('https://example.edu/api/v1',
                                 'path-to-asset/here')

        nuxeo_metadata = open(DIR_FIXTURES + '/nuxeo_doc_video.json').read()
        nuxeo_metadata = json.loads(nuxeo_metadata)
        isShownBy = h._get_isShownBy(nuxeo_metadata)

        self.assertEqual(
            isShownBy, 'https://s3.amazonaws.com/static.ucldc.cdlib.org/'
            'ucldc-nuxeo-thumb-media/4c80e254-6def-4230-9f28-bc48878568d4')

 def testHarvestControllerExists(self):
     httpretty.register_uri(
         httpretty.GET,
         "https://registry.cdlib.org/api/v1/collection/197/",
         body=open(DIR_FIXTURES + '/collection_api_test.json').read())
     httpretty.register_uri(
         httpretty.GET,
         re.compile("http://content.cdlib.org/oai?.*"),
         body=open(DIR_FIXTURES + '/testOAI-128-records.xml').read())
     collection = Collection(
         'https://registry.cdlib.org/api/v1/collection/197/')
     controller = fetcher.HarvestController(
         '[email protected]',
         collection,
         config_file=self.config_file,
         profile_path=self.profile_path)
     self.assertTrue(hasattr(controller, 'fetcher'))
     self.assertIsInstance(controller.fetcher, fetcher.OAIFetcher)
     self.assertTrue(hasattr(controller, 'campus_valid'))
     self.assertTrue(hasattr(controller, 'dc_elements'))
     self.assertTrue(hasattr(controller, 'datetime_start'))
     print(controller.s3path)
     self.assertEqual(controller.s3path,
                      'data-fetched/197/2017-07-14-1201/')
     shutil.rmtree(controller.dir_save)

 def testInit(self):
     '''Basic tdd start'''
     url = 'https://example.edu'
     playlist_id = 'testplaylist'
     page_size = 3
     url_first = fetcher.YouTube_Fetcher.url_playlistitems.format(
         api_key='boguskey',
         page_size=page_size,
         playlist_id=playlist_id,
         page_token='')
     httpretty.register_uri(
         httpretty.GET,
         url_first,
         body=open(DIR_FIXTURES + '/flickr-public-photos-1.xml').read())
     h = fetcher.YouTube_Fetcher(url, playlist_id, page_size=page_size)
     self.assertEqual(h.url_base, url)
     self.assertEqual(h.playlist_id, playlist_id)
     self.assertEqual(h.api_key, 'boguskey')
     self.assertEqual(h.page_size, page_size)
     self.assertEqual(h.playlistitems, {'nextPageToken': ''})
     self.assertEqual(
         h.url_playlistitems,
         'https://www.googleapis.com/youtube/v3/playlistItems'
         '?key={api_key}&maxResults={page_size}&part=contentDetails&'
         'playlistId={playlist_id}&pageToken={page_token}')
     self.assertEqual(
         h.url_video,
         'https://www.googleapis.com/youtube/v3/videos?'
         'key={api_key}&part=snippet&id={video_ids}'
         )

 def testLoggingMoreThan1000(self):
     httpretty.register_uri(
         httpretty.GET,
         "https://registry.cdlib.org/api/v1/collection/198/",
         body=open(DIR_FIXTURES + '/collection_api_big_test.json').read())
     httpretty.register_uri(
         httpretty.GET,
         re.compile("http://content.cdlib.org/oai?.*"),
         body=open(DIR_FIXTURES + '/testOAI-2400-records.xml').read())
     collection = Collection(
         'https://registry.cdlib.org/api/v1/collection/198/')
     controller = fetcher.HarvestController(
         '[email protected]',
         collection,
         config_file=self.config_file,
         profile_path=self.profile_path)
     controller.harvest()
     self.assertEqual(len(self.test_log_handler.records), 13)
     self.assertEqual(self.test_log_handler.formatted_records[1],
                      '[INFO] HarvestController: 100 records harvested')
     shutil.rmtree(controller.dir_save)
     self.assertEqual(self.test_log_handler.formatted_records[10],
                      '[INFO] HarvestController: 1000 records harvested')
     self.assertEqual(self.test_log_handler.formatted_records[11],
                      '[INFO] HarvestController: 2000 records harvested')
     self.assertEqual(self.test_log_handler.formatted_records[12],
                      '[INFO] HarvestController: 2400 records harvested')

 def test_single_fetching(self):
     url = 'http://single.edu'
     playlist_id = 'PLwtrWl_IBMJtjP5zMk6dVR-BRjzKqCPOM'
     url_vids = fetcher.YouTube_Fetcher.url_video
     httpretty.register_uri(
         httpretty.GET,
         url_vids,
         responses=[
             httpretty.Response(
                 body=open(DIR_FIXTURES + '/youtube_single_video.json').read(),
                 status=200)
         ])
     h = fetcher.YouTube_Fetcher(url, playlist_id)
     vids = []
     for v in h:
         vids.extend(v)
     self.assertEqual(len(vids), 1)
     self.assertEqual(vids[0], {
         u'contentDetails': {
             u'definition': u'sd',
             u'projection': u'rectangular',
             u'caption': u'false',
             u'duration': u'PT19M35S',
             u'licensedContent': True,
             u'dimension': u'2d'
         },
         u'kind': u'youtube#video',
         u'etag':
         u'"m2yskBQFythfE4irbTIeOgYYfBU/-3AtVAYcRLEynWZprpf0OGaY8zo"',
         u'id': u'0Yx8zrbsUu8'
     })