Java EndpointType.BIOCASE属性代码示例

@Test
public void testEquals() {
  Map<String, String> m1 = Maps.newHashMap();
  m1.put("foo", "bar");
  CrawlJob c1 = new CrawlJob(UUID.randomUUID(), EndpointType.BIOCASE, URI.create("http://www.foo.com"), 1, m1);

  Map<String, String> m2 = Maps.newHashMap();
  m2.put("foo", "bar");
  CrawlJob c2 = new CrawlJob(UUID.fromString(c1.getDatasetKey().toString()),
                             EndpointType.BIOCASE,
                             URI.create("http://www.foo.com"),
                             1,
                             m2);

  assertEquals(c1, c2);
  assertEquals(c1.hashCode(), c2.hashCode());
}
 

@Test
public void testNewAbcd206Single() {
  UUID datasetKey = UUID.randomUUID();
  OccurrenceSchemaType schemaType = OccurrenceSchemaType.ABCD_2_0_6;
  EndpointType protocol = EndpointType.BIOCASE;
  Integer crawlId = 1;
  fragmentProcessor.buildFragments(datasetKey, abcd206Single.getBytes(), schemaType, protocol, crawlId, null);

  Fragment got = fragmentPersistenceService.get(1);
  assertTrue(got.getKey() > 0);
  Calendar resultCal = Calendar.getInstance();
  resultCal.setTime(got.getHarvestedDate());
  Calendar cal = Calendar.getInstance();
  assertEquals(cal.get(Calendar.DAY_OF_YEAR), resultCal.get(Calendar.DAY_OF_YEAR));
  assertEquals(Fragment.FragmentType.XML, got.getFragmentType());
  assertEquals(schemaType, got.getXmlSchema());
  assertEquals(datasetKey, got.getDatasetKey());
  assertEquals(protocol, got.getProtocol());
  assertEquals(crawlId, got.getCrawlId());
  assertTrue(Arrays.equals(abcd206Single.getBytes(), got.getData()));
  assertTrue(Arrays.equals(DigestUtils.md5(abcd206Single), got.getDataHash()));
  assertNull(got.getUnitQualifier());
}
 

@Test
public void testUpdateFullXml() {
  Fragment orig = fragmentService.get(xmlKey);

  int crawlId = 567;
  Date harvestDate = new Date();
  byte[] xml = Bytes.toBytes("<parsing>this is not a love song</parsing>");
  byte[] xmlHash = DigestUtils.md5(xml);
  OccurrenceSchemaType xmlSchema = OccurrenceSchemaType.ABCD_2_0_6;
  EndpointType endpointType = EndpointType.BIOCASE;
  String unitQualifier = "Puma concolor";
  Long created = System.currentTimeMillis();

  Fragment update =
    new Fragment(orig.getDatasetKey(), xml, xmlHash, orig.getFragmentType(), endpointType, harvestDate, crawlId,
      xmlSchema, unitQualifier, created);
  update.setKey(orig.getKey());
  fragmentService.update(update);

  Fragment frag = fragmentService.get(xmlKey);
  assertNotNull(frag);
  assertEquals(XML_DATASET_KEY, frag.getDatasetKey());
  assertEquals(crawlId, frag.getCrawlId().intValue());
  assertEquals(unitQualifier, frag.getUnitQualifier());
  assertEquals(harvestDate, frag.getHarvestedDate());
  assertTrue(Arrays.equals(xml, frag.getData()));
  assertTrue(Arrays.equals(xmlHash, frag.getDataHash()));
  assertEquals(xmlSchema, frag.getXmlSchema());
  assertEquals(endpointType, frag.getProtocol());
  assertEquals(Fragment.FragmentType.XML, frag.getFragmentType());
  assertEquals(created, frag.getCreated());
}
 

@Test
public void testUpdateAbcd206OneOfMultiple() {
  // first create new
  UUID datasetKey = UUID.randomUUID();
  OccurrenceSchemaType schemaType = OccurrenceSchemaType.ABCD_2_0_6;
  EndpointType protocol = EndpointType.BIOCASE;
  Integer crawlId = 1;
  fragmentProcessor.buildFragments(datasetKey, abcd206Multi.getBytes(), schemaType, protocol, crawlId, null);

  // now attempt an "update"
  crawlId = 2;
  fragmentProcessor.buildFragments(datasetKey, abcd206MultiModified.getBytes(), schemaType, protocol, crawlId, null);

  Set<Fragment> fragments = Sets.newHashSet();
  Fragment first = fragmentPersistenceService.get(1);
  fragments.add(first);
  Fragment second = fragmentPersistenceService.get(2);
  fragments.add(second);

  int count = 0;
  for (Fragment got : fragments) {
    assertEquals(datasetKey, got.getDatasetKey());
    assertEquals(crawlId, got.getCrawlId());
    assertTrue(Arrays.equals(abcd206MultiModified.getBytes(), got.getData()));
    if ("Schistidium agassizii Sull. & Lesq. in Sull.".equals(got.getUnitQualifier())) {
      count++;
    } else if ("Grimmia alpicola Sw. ex Hedw.".equals(got.getUnitQualifier())) {
      count++;
    } else {
      fail("Unexpected UnitQualifier: " + got.getUnitQualifier());
    }
  }
  assertEquals(2, count);
}
 

@Test
public void testHBaseFailureDuringKeyLookup() {
  // make sure that a keylookup failure due to hbase failure does not result in same behaviour as "key not found"
  occurrenceKeyService = new OccurrenceKeyPersistenceService() {
    @Nullable
    @Override
    public KeyLookupResult findKey(Set<UniqueIdentifier> uniqueIdentifiers) {
      throw new ServiceUnavailableException("connection failure");
    }

    @Override
    public Set<Integer> findKeysByDataset(String datasetKey) {
      throw new UnsupportedOperationException("Not implemented yet");
    }

    @Override
    public KeyLookupResult generateKey(Set<UniqueIdentifier> uniqueIdentifiers) {
      throw new UnsupportedOperationException("Not implemented yet");
    }

    @Override
    public void deleteKey(int occurrenceKey, @Nullable String datasetKey) {
      throw new UnsupportedOperationException("Not implemented yet");
    }

    @Override
    public void deleteKeyByUniqueIdentifiers(Set<UniqueIdentifier> uniqueIdentifiers) {
      throw new UnsupportedOperationException("Not implemented yet");
    }
  };
  fragmentProcessor =
    new FragmentProcessor(fragmentPersistenceService, occurrenceKeyService, messagePublisher, zookeeperConnector);

  UUID datasetKey = UUID.randomUUID();
  OccurrenceSchemaType schemaType = OccurrenceSchemaType.ABCD_2_0_6;
  EndpointType protocol = EndpointType.BIOCASE;
  Integer crawlId = 1;
  fragmentProcessor.buildFragments(datasetKey, abcd206Single.getBytes(), schemaType, protocol, crawlId, null);

  Fragment got = fragmentPersistenceService.get(1);
  assertNull(got);
}
 

@Test
public void testEndToEndAbcd2() throws IOException, InterruptedException, URISyntaxException {
  UUID datasetKey = UUID.fromString(BOGART_DATASET_KEY);
  OccurrenceSchemaType xmlSchema = OccurrenceSchemaType.ABCD_2_0_6;
  Integer crawlId = 1;
  OccurrenceFragmentedMessage msg =
    new OccurrenceFragmentedMessage(datasetKey, crawlId, abcd206Single.getBytes(), xmlSchema, EndpointType.BIOCASE,
      null);
  messagePublisher.send(msg);

  TimeUnit.MILLISECONDS.sleep(10000);

  Occurrence got = occurrenceService.get(1);
  assertNotNull(got);
  assertEquals("BGBM", got.getVerbatimField(DwcTerm.institutionCode));
  assertEquals("AlgaTerra", got.getVerbatimField(DwcTerm.collectionCode));
  assertEquals("5834", got.getVerbatimField(DwcTerm.catalogNumber));
  assertEquals(datasetKey, got.getDatasetKey());
  assertEquals("Tetraedron caudatum (Corda) Hansgirg, 1888", got.getScientificName());
  assertEquals(52.123456, got.getDecimalLatitude().doubleValue(), 0.000001);
  assertEquals(13.123456, got.getDecimalLongitude().doubleValue(), 0.000001);
  assertEquals("WGS84", got.getGeodeticDatum());
  assertTrue(got.getIssues().contains(OccurrenceIssue.COORDINATE_REPROJECTED));
  assertEquals(450, got.getElevation().intValue());
  assertEquals(Country.fromIsoCode("DE"), got.getCountry());
  assertEquals("Kusber, W.-H.", got.getVerbatimField(DwcTerm.recordedBy));
  assertEquals("Nikolassee, Berlin", got.getVerbatimField(DwcTerm.locality));
  Calendar c = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
  c.set(1987, 3, 13);
  c.set(Calendar.HOUR_OF_DAY, 0);
  c.set(Calendar.MINUTE, 0);
  c.set(Calendar.SECOND, 0);
  c.set(Calendar.MILLISECOND, 0);
  assertEquals(c.getTime(), got.getEventDate());
  assertEquals(BasisOfRecord.HUMAN_OBSERVATION, got.getBasisOfRecord());
  assertEquals("Kusber, W.-H.", got.getVerbatimField(DwcTerm.identifiedBy));

  assertEquals(BGBM_KEY, got.getPublishingOrgKey().toString());
  assertEquals(Country.GERMANY, got.getPublishingCountry());
  assertEquals(EndpointType.BIOCASE, got.getProtocol());
  assertEquals("1", got.getVerbatimField(GbifTerm.gbifID));
  assertEquals(TypeStatus.HOLOTYPE, got.getTypeStatus());
  assertEquals(2, got.getMedia().size());
  assertEquals(new URI("http://www.tierstimmenarchiv.de/recordings/Ailuroedus_buccoides_V2010_04_short.mp3"),
    got.getMedia().get(0).getIdentifier());
}
 

@Test
public void testEndToEndZkCounts() throws IOException, InterruptedException {
  UUID datasetKey = UUID.randomUUID();

  // one abcd206
  OccurrenceSchemaType schemaType = OccurrenceSchemaType.ABCD_2_0_6;
  Integer crawlId = 1;
  OccurrenceFragmentedMessage msg =
    new OccurrenceFragmentedMessage(datasetKey, crawlId, abcd206Single.getBytes(), schemaType, EndpointType.BIOCASE,
      null);
  messagePublisher.send(msg);

  // two in one abcd 2
  schemaType = OccurrenceSchemaType.ABCD_2_0_6;
  crawlId = 1;
  msg =
    new OccurrenceFragmentedMessage(datasetKey, crawlId, abcd206Multi.getBytes(), schemaType, EndpointType.BIOCASE,
      null);
  messagePublisher.send(msg);

  // one dwc 1.4
  schemaType = OccurrenceSchemaType.DWC_1_4;
  crawlId = 1;
  msg =
    new OccurrenceFragmentedMessage(datasetKey, crawlId, dwc14.getBytes(), schemaType, EndpointType.BIOCASE, null);
  messagePublisher.send(msg);

  // dupe of dwc 1.4
  messagePublisher.send(msg);

  // update of dwc 1.4
  schemaType = OccurrenceSchemaType.DWC_1_4;
  crawlId = 1;
  msg =
    new OccurrenceFragmentedMessage(datasetKey, crawlId, dwc14_modified.getBytes(), schemaType, EndpointType.DIGIR,
      null);
  messagePublisher.send(msg);

  TimeUnit.MILLISECONDS.sleep(6000);

  assertEquals(5l,
    zookeeperConnector.readCounter(datasetKey, ZookeeperConnector.CounterName.FRAGMENT_RECEIVED).longValue());
  assertEquals(5l,
    zookeeperConnector.readCounter(datasetKey, ZookeeperConnector.CounterName.FRAGMENT_PROCESSED).longValue());
  assertEquals(0l,
    zookeeperConnector.readCounter(datasetKey, ZookeeperConnector.CounterName.RAW_OCCURRENCE_PERSISTED_ERROR)
      .longValue()
  );
  assertEquals(1l,
    zookeeperConnector.readCounter(datasetKey, ZookeeperConnector.CounterName.RAW_OCCURRENCE_PERSISTED_UNCHANGED)
      .longValue()
  );
  assertEquals(1l,
    zookeeperConnector.readCounter(datasetKey, ZookeeperConnector.CounterName.RAW_OCCURRENCE_PERSISTED_UPDATED)
      .longValue()
  );
  assertEquals(4l,
    zookeeperConnector.readCounter(datasetKey, ZookeeperConnector.CounterName.RAW_OCCURRENCE_PERSISTED_NEW)
      .longValue()
  );
  assertEquals(0l,
    zookeeperConnector.readCounter(datasetKey, ZookeeperConnector.CounterName.VERBATIM_OCCURRENCE_PERSISTED_ERROR)
      .longValue()
  );
  assertEquals(5l,
    zookeeperConnector.readCounter(datasetKey, ZookeeperConnector.CounterName.VERBATIM_OCCURRENCE_PERSISTED_SUCCESS)
      .longValue()
  );
  assertEquals(0l,
    zookeeperConnector.readCounter(datasetKey, ZookeeperConnector.CounterName.INTERPRETED_OCCURRENCE_PERSISTED_ERROR)
      .longValue()
  );
  assertEquals(5l, zookeeperConnector
    .readCounter(datasetKey, ZookeeperConnector.CounterName.INTERPRETED_OCCURRENCE_PERSISTED_SUCCESS).longValue());
}
 

@Test
public void testAbcd206() {
  OccurrenceSchemaType schema = OccurrenceSchemaType.ABCD_2_0_6;
  UUID datasetKey = UUID.randomUUID();
  Fragment frag = new Fragment(datasetKey, abcd206Single.getBytes(), DigestUtils.md5(abcd206Single.getBytes()),
    Fragment.FragmentType.XML, EndpointType.BIOCASE, new Date(), 1, schema, null, null);
  frag.setKey(1);

  VerbatimOccurrence got = FragmentParser.parse(frag);
  assertNotNull(got);
  assertEquals("BGBM", got.getVerbatimField(DwcTerm.institutionCode));
  assertEquals("AlgaTerra", got.getVerbatimField(DwcTerm.collectionCode));
  assertEquals("5834", got.getVerbatimField(DwcTerm.catalogNumber));
  assertEquals(datasetKey, got.getDatasetKey());
  assertNull(got.getVerbatimField(GbifInternalTerm.unitQualifier));

  assertEquals(1, got.getKey().intValue());
  assertEquals("Tetraedron caudatum (Corda) Hansg.", got.getVerbatimField(DwcTerm.scientificName));
  assertEquals("52.123456", got.getVerbatimField(DwcTerm.decimalLatitude));
  assertEquals("13.123456", got.getVerbatimField(DwcTerm.decimalLongitude));
  assertEquals("50", got.getVerbatimField(DwcTerm.coordinateUncertaintyInMeters));
  assertEquals("400", got.getVerbatimField(DwcTerm.minimumElevationInMeters));
  assertEquals("500", got.getVerbatimField(DwcTerm.maximumElevationInMeters));
  assertEquals("DE", got.getVerbatimField(DwcTerm.country));
  assertEquals("Kusber, W.-H.", got.getVerbatimField(DwcTerm.recordedBy));
  assertEquals("Nikolassee, Berlin", got.getVerbatimField(DwcTerm.locality));
  assertEquals("1987-04-13T00:00:00", got.getVerbatimField(DwcTerm.eventDate));
  assertEquals("HumanObservation", got.getVerbatimField(DwcTerm.basisOfRecord));
  assertEquals("Kusber, W.-H.", got.getVerbatimField(DwcTerm.identifiedBy));
  assertEquals("Holotype", got.getVerbatimField(DwcTerm.typeStatus));
  assertEquals("Tetraedron caudatum (Corda) Hansg.", got.getVerbatimField(GbifTerm.typifiedName));

  assertNotNull(got.getExtensions().get(Extension.MULTIMEDIA));
  List<Map<Term,String>> mediaObjects = got.getExtensions().get(Extension.MULTIMEDIA);
  assertEquals(2, mediaObjects.size());
  Map<Term,String> medium = mediaObjects.get(0);
  assertEquals("http://www.tierstimmenarchiv.de/recordings/Ailuroedus_buccoides_V2010_04_short.mp3",
    medium.get(DcTerm.identifier));
  assertEquals("http://www.tierstimmenarchiv.de/webinterface/contents/showdetails.php?edit=-1&unique_id=TSA:Ailuroedus_buccoides_V_2010_4_1&autologin=true",
    medium.get(DcTerm.references));
  assertEquals("audio/mp3", medium.get(DcTerm.format));
  assertEquals("CC BY-NC-ND (Attribution for non commercial use only and without derivative)",
    medium.get(DcTerm.license));

}