Golang models.GenericFile类代码示例

// GenericFileSave saves a Generic File record to Pharos. If the Generic
// File's ID is zero, this performs a POST to create a new record.
// For non-zero IDs, this performs a PUT to update the existing record.
// Either way, the record must have an IntellectualObject ID. The response
// object will have a new copy of the GenericFile if the save was successful.
func (client *PharosClient) GenericFileSave(obj *models.GenericFile) *PharosResponse {
	// Set up the response object
	resp := NewPharosResponse(PharosGenericFile)
	resp.files = make([]*models.GenericFile, 1)

	// URL and method
	relativeUrl := fmt.Sprintf("/api/%s/files/", client.apiVersion)
	httpMethod := "POST"
	if obj.Id > 0 {
		// PUT URL looks like /api/v2/files/college.edu%2Fobject_name%2Ffile.xml
		relativeUrl = fmt.Sprintf("%s%s", relativeUrl, escapeSlashes(obj.Identifier))
		httpMethod = "PUT"
	}
	absoluteUrl := client.BuildUrl(relativeUrl)

	// Prepare the JSON data
	postData, err := obj.SerializeForPharos()
	if err != nil {
		resp.Error = err
	}

	// Run the request
	client.DoRequest(resp, httpMethod, absoluteUrl, bytes.NewBuffer(postData))
	if resp.Error != nil {
		return resp
	}

	// Parse the JSON from the response body
	gf := &models.GenericFile{}
	resp.Error = json.Unmarshal(resp.data, gf)
	if resp.Error == nil {
		resp.files[0] = gf
	}
	return resp
}

// buildFile saves a data file from the tar archive to disk,
// then returns a struct with data we'll need to construct the
// GenericFile object in Fedora later.
func (reader *Reader) saveWithChecksums(gf *models.GenericFile) {
	// Set up a MultiWriter to stream data ONCE to file,
	// md5 and sha256. We don't want to process the stream
	// three separate times.
	err := os.MkdirAll(filepath.Dir(gf.IngestLocalPath), 0755)
	if err != nil {
		gf.IngestErrorMessage = err.Error()
		return
	}
	outputWriter, err := os.OpenFile(gf.IngestLocalPath, os.O_CREATE|os.O_WRONLY, 0644)
	if outputWriter != nil {
		defer outputWriter.Close()
	}
	if err != nil {
		gf.IngestErrorMessage = fmt.Sprintf("Error opening %s for writing: %v", gf.IngestLocalPath, err)
		return
	}
	md5Hash := md5.New()
	shaHash := sha256.New()
	multiWriter := io.MultiWriter(md5Hash, shaHash, outputWriter)
	io.Copy(multiWriter, reader.tarReader)
	gf.IngestMd5 = fmt.Sprintf("%x", md5Hash.Sum(nil))
	gf.IngestSha256 = fmt.Sprintf("%x", shaHash.Sum(nil))
	gf.IngestSha256GeneratedAt = time.Now().UTC()
	gf.FileFormat, _ = platform.GuessMimeType(gf.IngestLocalPath) // on err, defaults to application/binary
	return
}

// Initializes the uploader object with connection data and metadata
// for this specific GenericFile.
func (storer *APTStorer) initUploader(ingestState *models.IngestState, gf *models.GenericFile, sendWhere string) *network.S3Upload {
	var region string
	var bucket string
	if sendWhere == "s3" {
		region = storer.Context.Config.APTrustS3Region
		bucket = storer.Context.Config.PreservationBucket
	} else if sendWhere == "glacier" {
		region = storer.Context.Config.APTrustGlacierRegion
		bucket = storer.Context.Config.ReplicationBucket
	} else {
		ingestState.IngestManifest.StoreResult.AddError("Cannot save %s to %s because "+
			"storer doesn't know where %s is", gf.Identifier, sendWhere)
		ingestState.IngestManifest.StoreResult.ErrorIsFatal = true
		return nil
	}
	uploader := network.NewS3Upload(
		region,
		bucket,
		gf.IngestUUID,
		gf.FileFormat,
	)
	uploader.AddMetadata("institution", ingestState.IngestManifest.Object.Institution)
	uploader.AddMetadata("bag", ingestState.IngestManifest.Object.Identifier)
	uploader.AddMetadata("bagpath", gf.OriginalPath())
	uploader.AddMetadata("md5", gf.IngestMd5)
	uploader.AddMetadata("sha256", gf.IngestSha256)
	return uploader
}

func TestInstitutionIdentifier(t *testing.T) {
	genericFile := models.GenericFile{}
	genericFile.Identifier = "uc.edu/cin.675812/data/object.properties"
	instId, err := genericFile.InstitutionIdentifier()
	if err != nil {
		t.Errorf(err.Error())
		return
	}
	assert.Equal(t, "uc.edu", instId)
}

func (storer *APTStorer) markFileAsStored(gf *models.GenericFile, sendWhere, storageUrl string) {
	if sendWhere == "s3" {
		gf.IngestStoredAt = time.Now().UTC()
		gf.IngestStorageURL = storageUrl
		gf.URI = storageUrl
		events := gf.FindEventsByType(constants.EventIdentifierAssignment)
		var event *models.PremisEvent
		for i := range events {
			existingEvent := events[i]
			if strings.HasPrefix(existingEvent.OutcomeDetail, "http://") ||
				strings.HasPrefix(existingEvent.OutcomeDetail, "https://") {
				event = existingEvent
				break
			}
		}
		if event != nil {
			event.DateTime = time.Now().UTC()
		}
	} else if sendWhere == "glacier" {
		gf.IngestReplicatedAt = time.Now().UTC()
		gf.IngestReplicationURL = storageUrl
		events := gf.FindEventsByType(constants.EventReplication)
		if events != nil && len(events) > 0 {
			events[0].DateTime = time.Now().UTC()
		}
	}
}

// Returns a reader that can read the file from within the tar archive.
// The S3 uploader uses this reader to stream data to S3 and Glacier.
func (storer *APTStorer) getReadCloser(ingestState *models.IngestState, gf *models.GenericFile) (*fileutil.TarFileIterator, io.ReadCloser) {
	tarFilePath := ingestState.IngestManifest.Object.IngestTarFilePath
	tfi, err := fileutil.NewTarFileIterator(tarFilePath)
	if err != nil {
		msg := fmt.Sprintf("Can't get TarFileIterator for %s: %v", tarFilePath, err)
		ingestState.IngestManifest.StoreResult.AddError(msg)
		return nil, nil
	}
	origPathWithBagName, err := gf.OriginalPathWithBagName()
	if err != nil {
		ingestState.IngestManifest.StoreResult.AddError(err.Error())
		return nil, nil
	}
	readCloser, err := tfi.Find(origPathWithBagName)
	if err != nil {
		msg := fmt.Sprintf("Can't get reader for %s: %v", gf.Identifier, err)
		ingestState.IngestManifest.StoreResult.AddError(msg)
		if readCloser != nil {
			readCloser.Close()
		}
		return nil, nil
	}
	return tfi, readCloser
}

func TestPreservationStorageFileName(t *testing.T) {
	genericFile := models.GenericFile{}
	genericFile.URI = ""
	fileName, err := genericFile.PreservationStorageFileName()
	if err == nil {
		t.Errorf("PreservationStorageFileName() should have returned an error")
	}
	genericFile.URI = "https://s3.amazonaws.com/aptrust.test.preservation/a58a7c00-392f-11e4-916c-0800200c9a66"
	fileName, err = genericFile.PreservationStorageFileName()
	if err != nil {
		t.Errorf("PreservationStorageFileName() returned an error: %v", err)
		return
	}
	assert.Equal(t, "a58a7c00-392f-11e4-916c-0800200c9a66", fileName)
}

func TestOriginalPathWithBagName(t *testing.T) {
	genericFile := models.GenericFile{}
	genericFile.IntellectualObjectIdentifier = "uc.edu/cin.675812"

	// Top-level custom tag file
	genericFile.Identifier = "uc.edu/cin.675812/tagmanifest-sha256.txt"
	origPath, err := genericFile.OriginalPathWithBagName()
	require.Nil(t, err)
	assert.Equal(t, "cin.675812/tagmanifest-sha256.txt", origPath)

	// Payload file
	genericFile.Identifier = "uc.edu/cin.675812/data/object.properties"
	origPath, err = genericFile.OriginalPathWithBagName()
	require.Nil(t, err)
	assert.Equal(t, "cin.675812/data/object.properties", origPath)

	// Nested custom tag file
	genericFile.Identifier = "uc.edu/cin.675812/custom/tag/dir/special_info.xml"
	origPath, err = genericFile.OriginalPathWithBagName()
	require.Nil(t, err)
	assert.Equal(t, "cin.675812/custom/tag/dir/special_info.xml", origPath)
}

func (storer *APTStorer) saveFile(ingestState *models.IngestState, gf *models.GenericFile) {
	existingSha256, err := storer.getExistingSha256(gf.Identifier)
	if err != nil {
		storer.Context.MessageLog.Error(err.Error())
		ingestState.IngestManifest.StoreResult.AddError(err.Error())
		return
	}
	// Set this, for the record.
	if existingSha256 != nil {
		gf.IngestPreviousVersionExists = true
		gf.Id = existingSha256.GenericFileId

		uuid, err := storer.getUuidOfExistingFile(gf.Identifier)
		if err != nil {
			message := fmt.Sprintf("Cannot find existing UUID for %s: %v", gf.Identifier, err.Error())
			ingestState.IngestManifest.StoreResult.AddError(message)
			storer.Context.MessageLog.Error(message)
			// Probably not fatal, but treat it as such for now,
			// because we don't want leave orphan objects in S3,
			// or have the GenericFile.URL not match the actual
			// storage URL. This should only happen if a depositor
			// deletes the existing version of a GenericFile while
			// we are processing this ingest. The window for that
			// to happen is usually between a few seconds and a few
			// hours.
			ingestState.IngestManifest.StoreResult.ErrorIsFatal = true
			return
		}
		if uuid == "" {
			message := fmt.Sprintf("Cannot find existing UUID for %s.", gf.Identifier)
			ingestState.IngestManifest.StoreResult.AddError(message)
			storer.Context.MessageLog.Error(message)
			// Probably not fatal, but treat it as such for now.
			// Same note as in previous if statement above.
			ingestState.IngestManifest.StoreResult.ErrorIsFatal = true
			return
		} else {
			// OK. Set the GenericFile's UUID to match the existing file's
			// UUID, so that we overwrite the existing file, and so the
			// GenericFile record in Pharos still has the correct URL.
			message := fmt.Sprintf("Resetting UUID for '%s' to '%s' so we can overwrite "+
				"the currently stored version of the file.",
				gf.Identifier, uuid)
			storer.Context.MessageLog.Info(message)
			// TODO: Test this in integration post test.
			gf.IngestUUID = uuid
		}

		if existingSha256.Digest != gf.IngestSha256 {
			storer.Context.MessageLog.Info(
				"GenericFile %s has same sha256. Does not need save.", gf.Identifier)
			gf.IngestNeedsSave = false
		}
	}
	// Now copy to storage only if the file has changed.
	if gf.IngestNeedsSave {
		storer.Context.MessageLog.Info("File %s needs save", gf.Identifier)
		if gf.IngestStoredAt.IsZero() || gf.IngestStorageURL == "" {
			storer.copyToLongTermStorage(ingestState, gf, "s3")
		}
		if gf.IngestReplicatedAt.IsZero() || gf.IngestReplicationURL == "" {
			storer.copyToLongTermStorage(ingestState, gf, "glacier")
		}
	}
}