Java EndpointType.DWC_ARCHIVE属性代码示例

@Before
public void setUp() throws Exception {
  long now = new Date().getTime();
  zkServer = new TestingServer();
  curator = CuratorFrameworkFactory.builder().connectString(zkServer.getConnectString()).namespace("crawler")
    .retryPolicy(new RetryNTimes(1, 1000)).build();
  curator.start();
  zookeeperConnector = new ZookeeperConnector(curator);

  fragmentService = new FragmentPersistenceServiceMock(new OccurrenceKeyPersistenceServiceMock());
  String json = Resources.toString(Resources.getResource("fragment.json"), org.apache.commons.io.Charsets.UTF_8);
  Fragment fragment = new Fragment(DATASET_KEY, json.getBytes(Charsets.UTF_8), json.getBytes(Charsets.UTF_8),
    Fragment.FragmentType.JSON, EndpointType.DWC_ARCHIVE, new Date(), 1, null, null, new Date().getTime());
  Set<UniqueIdentifier> uniqueIds = Sets.newHashSet();
  uniqueIds.add(new HolyTriplet(DATASET_KEY, "ic", "cc", "cn", null));
  fragmentService.insert(fragment, uniqueIds);
  occurrenceService = new OccurrencePersistenceServiceMock(fragmentService);
  verbatimProcessor = new VerbatimProcessor(fragmentService, occurrenceService, messagePublisher, zookeeperConnector);
  FragmentPersistedListener fragmentPersistedListener = new FragmentPersistedListener(verbatimProcessor);
  MessageListener messageListener = new MessageListener(CONNECTION_PARAMETERS);
  messageListener.listen("frag_persisted_test_" + now, 1, fragmentPersistedListener);
}
 

@Before
public void setUp() throws Exception {
  ApiClientConfiguration cfg = new ApiClientConfiguration();;
  cfg.url = URI.create("http://api.gbif-dev.org/v1/");

  FragmentPersistenceService fragmentPersister =
    new FragmentPersistenceServiceMock(new OccurrenceKeyPersistenceServiceMock());
  Fragment fragment = new Fragment(DATASET_KEY, "fake".getBytes(Charsets.UTF_8), "fake".getBytes(Charsets.UTF_8),
    Fragment.FragmentType.JSON, EndpointType.DWC_ARCHIVE, new Date(), 1, null, null, new Date().getTime());
  Set<UniqueIdentifier> uniqueIds = Sets.newHashSet();
  uniqueIds.add(new HolyTriplet(DATASET_KEY, "ic", "cc", "cn", null));
  fragmentPersister.insert(fragment, uniqueIds);
  interpreter = new OccurrenceInterpreter(new DatasetInfoInterpreter(cfg.newApiClient()),
    new TaxonomyInterpreter(cfg.newApiClient()),
    new LocationInterpreter(new CoordinateInterpreter(cfg.newApiClient())));

  verb = buildVerbatim(fragment.getKey());

  verbMod = buildVerbatim(fragment.getKey());
  verbMod.setVerbatimField(DwcTerm.scientificName, "Panthera onca goldmani");
}
 

@Test
public void testInsertEmptyIds() {
  byte[] data = "boo".getBytes();
  byte[] dataHash = "far".getBytes();
  Fragment frag =
    new Fragment(UUID.randomUUID(), data, dataHash, Fragment.FragmentType.JSON, EndpointType.DWC_ARCHIVE, new Date(),
      1, null, null, null);
  exception.expect(NullPointerException.class);
  exception.expectMessage("uniqueIds can't be null");
  fragmentService.insert(frag, null);
}
 

@Test
public void testEndToEndDwca() throws IOException, InterruptedException, URISyntaxException {
  UUID datasetKey = UUID.fromString(PONTAURUS_DATASET_KEY);
  OccurrenceSchemaType xmlSchema = OccurrenceSchemaType.DWCA;
  Integer crawlId = 1;

  DwcaValidationReport report = new DwcaValidationReport(datasetKey,
    new OccurrenceValidationReport(1, 1, 0, 1, 0, true));
  OccurrenceFragmentedMessage msg =
    new OccurrenceFragmentedMessage(datasetKey, crawlId, dwcaSingle.getBytes(), xmlSchema, EndpointType.DWC_ARCHIVE,
      report);
  messagePublisher.send(msg);

  TimeUnit.MILLISECONDS.sleep(5000);

  Occurrence got = occurrenceService.get(1);
  assertNotNull(got);
  assertEquals("BGBM", got.getVerbatimField(DwcTerm.institutionCode));
  assertEquals("Pontaurus", got.getVerbatimField(DwcTerm.collectionCode));
  assertEquals("988", got.getVerbatimField(DwcTerm.catalogNumber));
  assertEquals(datasetKey, got.getDatasetKey());
  // note: this is set here inside the occ project, but from a ws call the serializer will omit these 'superseded' terms
  assertEquals("Verbascum cheiranthifolium var. cheiranthifolium", got.getVerbatimField(DwcTerm.scientificName));
  assertEquals("Verbascum cheiranthifolium var. cheiranthifolium", got.getScientificName());
  assertEquals(37.421230, got.getDecimalLatitude().doubleValue(), 0.000001);
  assertEquals(34.568123, got.getDecimalLongitude().doubleValue(), 0.000001);
  assertEquals(Country.fromIsoCode("TR"), got.getCountry());
  Calendar c = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
  c.set(1999, 6, 30);
  c.set(Calendar.HOUR_OF_DAY, 0);
  c.set(Calendar.MINUTE, 0);
  c.set(Calendar.SECOND, 0);
  c.set(Calendar.MILLISECOND, 0);
  assertEquals(c.getTime(), got.getEventDate());
  assertEquals(BasisOfRecord.PRESERVED_SPECIMEN, got.getBasisOfRecord());
  assertEquals("Markus Döring", got.getVerbatimField(DwcTerm.identifiedBy));

  assertEquals(BGBM_KEY, got.getPublishingOrgKey().toString());
  assertEquals(Country.GERMANY, got.getPublishingCountry());
  assertEquals(EndpointType.DWC_ARCHIVE, got.getProtocol());
  assertEquals("1", got.getVerbatimField(GbifTerm.gbifID));
  assertEquals("ABC123", got.getVerbatimField(DwcTerm.occurrenceID));

  // multimedia
  assertNotNull(got.getMedia());
  assertEquals(1, got.getMedia().size());
  assertEquals(MediaType.StillImage, got.getMedia().get(0).getType());
  assertEquals("http://digit.snm.ku.dk/www/Aves/full/AVES-100348_Caprimulgus_pectoralis_fervidus_ad____f.jpg",
    got.getMedia().get(0).getIdentifier().toString());

}
 

@Test
public void testParsePreferred() throws IOException {
  UUID datasetKey = UUID.randomUUID();
  String json = Resources.toString(Resources.getResource("fragment.json"), Charsets.UTF_8);
  Fragment fragment = new Fragment(datasetKey, json.getBytes("UTF-8"), DigestUtils.md5(json.getBytes("UTF-8")),
    Fragment.FragmentType.JSON, EndpointType.DWC_ARCHIVE, new Date(), 1,
    OccurrenceSchemaType.DWCA, null, null);
  VerbatimOccurrence verb = JsonFragmentParser.parseRecord(fragment);
  assertNotNull(verb);

  assertEquals("Plantae", verb.getVerbatimField(DwcTerm.kingdom));
  assertNull(verb.getVerbatimField(DwcTerm.scientificNameAuthorship));
  assertEquals("2400", verb.getVerbatimField(DwcTerm.minimumElevationInMeters));
  assertEquals("Magnoliophyta", verb.getVerbatimField(DwcTerm.phylum));
  assertEquals("BGBM", verb.getVerbatimField(DwcTerm.institutionCode));
  assertEquals("specimen", verb.getVerbatimField(DwcTerm.basisOfRecord));
  assertEquals("Verbascum cheiranthifolium var. cheiranthifolium", verb.getVerbatimField(DwcTerm.scientificName));
  assertEquals("Pontaurus", verb.getVerbatimField(DwcTerm.collectionCode));
  assertEquals("Markus Döring", verb.getVerbatimField(DwcTerm.recordedBy));
  assertNull(verb.getVerbatimField(DwcTerm.locality));
  assertEquals("7", verb.getVerbatimField(DwcTerm.month));
  assertEquals("988", verb.getVerbatimField(DwcTerm.catalogNumber));
  assertEquals("37.42123", verb.getVerbatimField(DwcTerm.decimalLatitude));
  assertEquals("1999", verb.getVerbatimField(DwcTerm.year));
  assertEquals("Verbascum", verb.getVerbatimField(DwcTerm.genus));
  assertEquals("Markus Döring", verb.getVerbatimField(DwcTerm.identifiedBy));
  assertEquals("Scrophulariales", verb.getVerbatimField(DwcTerm.order));
  assertEquals("30", verb.getVerbatimField(DwcTerm.day));
  assertEquals("Fake", verb.getVerbatimField(DwcTerm.country));
  assertEquals("TR", verb.getVerbatimField(DwcTerm.countryCode));
  assertEquals("34.5681234", verb.getVerbatimField(DwcTerm.decimalLongitude));
  assertEquals("Magnoliopsida", verb.getVerbatimField(DwcTerm.class_));
  assertEquals("Scrophulariaceae", verb.getVerbatimField(DwcTerm.family));
  assertEquals("Asia", verb.getVerbatimField(DwcTerm.continent));
  assertNull(verb.getVerbatimField(TermFactory.instance().findTerm("extensions")));

  // test image extension
  for (Extension ext : Extension.values()) {
    if (ext == Extension.IMAGE) continue;
    assertFalse(verb.getExtensions().containsKey(ext));
  }
  assertTrue(verb.getExtensions().containsKey(Extension.IMAGE));
  assertEquals(1, verb.getExtensions().get(Extension.IMAGE).size());
  assertEquals("http://digit.snm.ku.dk/www/Aves/full/AVES-100348_Caprimulgus_pectoralis_fervidus_ad____f.jpg", verb.getExtensions().get(Extension.IMAGE).get(0).get(DcTerm.identifier));
}
 

@Test
public void testExtensions() throws IOException {
    UUID datasetKey = UUID.randomUUID();
    String json = Resources.toString(Resources.getResource("fragment-extensions.json"), Charsets.UTF_8);
    Fragment fragment = new Fragment(datasetKey, json.getBytes("UTF-8"), DigestUtils.md5(json.getBytes("UTF-8")),
            Fragment.FragmentType.JSON, EndpointType.DWC_ARCHIVE, new Date(), 1,
            OccurrenceSchemaType.DWCA, null, null);
    VerbatimOccurrence verb = JsonFragmentParser.parseRecord(fragment);
    assertNotNull(verb);

    assertEquals("http://collections.mnh.si.edu/media/index.php?irn=10842031", verb.getVerbatimField(DwcTerm.associatedMedia));
    // test extensions
    //TODO: The EOL extension is not recognized as the dwc-api dependency is outdated and does not contain EOL terms...
    // once updated please change the size below to 3 and outcomment the eol assertions
    assertEquals(2, verb.getExtensions().size());
    // test media extension
    assertEquals(2, verb.getExtensions().get(Extension.MULTIMEDIA).size());
    Map<Term, String> m = verb.getExtensions().get(Extension.MULTIMEDIA).get(0);
    assertEquals("http://www.mnh.si.edu/rc/db/2data_access_policy.html", m.get(DcTerm.license));
    assertEquals("USNMENT832289", m.get(DcTerm.description));
    assertEquals("National Museum of Natural History, Smithsonian Institution", m.get(DcTerm.publisher));
    assertEquals("http://collections.mnh.si.edu/media/index.php?irn=10842031", m.get(DcTerm.identifier));
    assertEquals("Trichardis picta", m.get(DcTerm.title));
    assertEquals("Image", m.get(DcTerm.type));
    assertEquals("National Museum of Natural History, Smithsonian Institution", m.get(DcTerm.rightsHolder));
    assertEquals("general public", m.get(DcTerm.audience));
    m = verb.getExtensions().get(Extension.MULTIMEDIA).get(1);
    assertEquals("http://collections.mnh.si.edu/media/index.php?irn=10842031b", m.get(DcTerm.identifier));
    // test EOL extension
    //TODO: outcomment once dwc-api is updated
    //assertEquals(1, verb.getExtensions().get(Extension.EOL_MEDIA).size());
    //m = verb.getExtensions().get(Extension.EOL_MEDIA).get(0);
    //assertEquals("http://www.mnh.si.edu/rc/db/2data_access_policy.html", m.get(DcTerm.license));
    //assertEquals("USNMENT832289", m.get(DcTerm.description));
    //assertEquals("National Museum of Natural History, Smithsonian Institution", m.get(DcTerm.publisher));
    //assertEquals("http://collections.mnh.si.edu/media/index.php?irn=10842031c", m.get(DcTerm.identifier));
    //assertEquals("Trichardis picta", m.get(DcTerm.title));
    //assertEquals("Image", m.get(DcTerm.type));
    //assertEquals("National Museum of Natural History, Smithsonian Institution", m.get(DcTerm.rightsHolder));
    //assertEquals("general public", m.get(DcTerm.audience));
    // test identified extension
    assertEquals(1, verb.getExtensions().get(Extension.IDENTIFICATION).size());
    m = verb.getExtensions().get(Extension.IDENTIFICATION).get(0);
    assertEquals("Marta marta", m.get(DwcTerm.scientificName));
    assertEquals("T Bone", m.get(DwcTerm.identifiedBy));
}