Golang database.Layer类代码示例

func detectOS(data map[string][]byte, parent *database.Layer) (detectedOS string, err error) {
	detectedOS = detectors.DetectOS(data)

	// Attempt to detect the OS from the parent layer.
	if detectedOS == "" && parent != nil {
		detectedOS, err = parent.OperatingSystem()
		if err != nil {
			return "", err
		}
	}

	// If the detectedOS is not in the supported OS list, the OS is unsupported.
	if detectedOS != "" {
		isSupported := false
		for _, osPrefix := range SupportedOS {
			if strings.HasPrefix(detectedOS, osPrefix) {
				isSupported = true
				break
			}
		}
		if !isSupported {
			return "", ErrUnsupported
		}
	}

	return
}

// Process detects the OS of a layer, the packages it installs/removes, and
// then stores everything in the database.
func Process(ID, parentID, path string) error {
	if ID == "" {
		return cerrors.NewBadRequestError("could not process a layer which does not have ID")
	}
	if path == "" {
		return cerrors.NewBadRequestError("could not process a layer which does not have a path")
	}

	log.Debugf("layer %s: processing (Location: %s, Engine version: %d, Parent: %s)", ID, utils.CleanURL(path), Version, parentID)

	// Check to see if the layer is already in the database.
	layer, err := database.FindOneLayerByID(ID, []string{database.FieldLayerEngineVersion})
	if err != nil && err != cerrors.ErrNotFound {
		return err
	}

	var parent *database.Layer

	if layer != nil {
		// The layer is already in the database, check if we need to update it.
		if layer.EngineVersion >= Version {
			log.Debugf("layer %s: layer content has already been processed in the past with engine %d. Current engine is %d. skipping analysis", ID, layer.EngineVersion, Version)
			return nil
		}

		log.Debugf("layer %s: layer content has been analyzed in the past with engine %d. Current engine is %d. analyzing again", ID, layer.EngineVersion, Version)
	} else {
		// The layer is a new one, create a base struct that we will fill.
		layer = &database.Layer{ID: ID, EngineVersion: Version}

		// Check to make sure that the parent's layer has already been processed.
		if parentID != "" {
			parent, err = database.FindOneLayerByID(parentID, []string{database.FieldLayerOS, database.FieldLayerPackages, database.FieldLayerPackages})
			if err != nil && err != cerrors.ErrNotFound {
				return err
			}
			if parent == nil {
				log.Warningf("layer %s: the parent layer (%s) is unknown. it must be processed first", ID, parentID)
				return ErrParentUnknown
			}
			layer.ParentNode = parent.GetNode()
		}
	}

	// Analyze the content.
	layer.OS, layer.InstalledPackagesNodes, layer.RemovedPackagesNodes, err = detectContent(ID, path, parent)
	if err != nil {
		return err
	}

	return database.InsertLayer(layer)
}

// detectAndInsertInstalledAndRemovedPackages finds the installed and removed
// package nodes and inserts the installed packages into the database.
func detectAndInsertInstalledAndRemovedPackages(detectedOS string, packageList []*database.Package, parent *database.Layer) (installedNodes, removedNodes []string, err error) {
	// Get the parent layer's packages.
	parentPackageNodes, err := parent.AllPackages()
	if err != nil {
		return nil, nil, err
	}
	parentPackages, err := database.FindAllPackagesByNodes(parentPackageNodes, []string{database.FieldPackageName, database.FieldPackageVersion})
	if err != nil {
		return nil, nil, err
	}

	// Map detected packages (name:version) string to packages.
	packagesNVMapToPackage := make(map[string]*database.Package)
	for _, p := range packageList {
		packagesNVMapToPackage[p.Name+":"+p.Version.String()] = p
	}

	// Map parent's packages (name:version) string to nodes.
	parentPackagesNVMapToNodes := make(map[string]string)
	for _, p := range parentPackages {
		parentPackagesNVMapToNodes[p.Name+":"+p.Version.String()] = p.Node
	}

	// Build a list of the parent layer's packages' node values.
	var parentPackagesNV []string
	for _, p := range parentPackages {
		parentPackagesNV = append(parentPackagesNV, p.Name+":"+p.Version.String())
	}

	// Build a list of the layer packages' node values.
	var layerPackagesNV []string
	for _, p := range packageList {
		layerPackagesNV = append(layerPackagesNV, p.Name+":"+p.Version.String())
	}

	// Calculate the installed and removed packages.
	removedPackagesNV := utils.CompareStringLists(parentPackagesNV, layerPackagesNV)
	installedPackagesNV := utils.CompareStringLists(layerPackagesNV, parentPackagesNV)

	// Build a list of all the installed packages.
	var installedPackages []*database.Package
	for _, nv := range installedPackagesNV {
		p, _ := packagesNVMapToPackage[nv]
		p.OS = detectedOS
		installedPackages = append(installedPackages, p)
	}

	// Insert that list into the database.
	err = database.InsertPackages(installedPackages)
	if err != nil {
		return nil, nil, err
	}

	// Build the list of installed package nodes.
	for _, p := range installedPackages {
		if p.Node != "" {
			installedNodes = append(installedNodes, p.Node)
		}
	}

	// Build the list of removed package nodes.
	for _, nv := range removedPackagesNV {
		node, _ := parentPackagesNVMapToNodes[nv]
		removedNodes = append(removedNodes, node)
	}

	return
}

func (pgSQL *pgSQL) FindLayer(name string, withFeatures, withVulnerabilities bool) (database.Layer, error) {
	subquery := "all"
	if withFeatures {
		subquery += "/features"
	} else if withVulnerabilities {
		subquery += "/features+vulnerabilities"
	}
	defer observeQueryTime("FindLayer", subquery, time.Now())

	// Find the layer
	var layer database.Layer
	var parentID zero.Int
	var parentName zero.String
	var namespaceID zero.Int
	var namespaceName sql.NullString

	t := time.Now()
	err := pgSQL.QueryRow(searchLayer, name).
		Scan(&layer.ID, &layer.Name, &layer.EngineVersion, &parentID, &parentName, &namespaceID,
			&namespaceName)
	observeQueryTime("FindLayer", "searchLayer", t)

	if err != nil {
		return layer, handleError("searchLayer", err)
	}

	if !parentID.IsZero() {
		layer.Parent = &database.Layer{
			Model: database.Model{ID: int(parentID.Int64)},
			Name:  parentName.String,
		}
	}
	if !namespaceID.IsZero() {
		layer.Namespace = &database.Namespace{
			Model: database.Model{ID: int(namespaceID.Int64)},
			Name:  namespaceName.String,
		}
	}

	// Find its features
	if withFeatures || withVulnerabilities {
		// Create a transaction to disable hash/merge joins as our experiments have shown that
		// PostgreSQL 9.4 makes bad planning decisions about:
		// - joining the layer tree to feature versions and feature
		// - joining the feature versions to affected/fixed feature version and vulnerabilities
		// It would for instance do a merge join between affected feature versions (300 rows, estimated
		// 3000 rows) and fixed in feature version (100k rows). In this case, it is much more
		// preferred to use a nested loop.
		tx, err := pgSQL.Begin()
		if err != nil {
			return layer, handleError("FindLayer.Begin()", err)
		}
		defer tx.Commit()

		_, err = tx.Exec(disableHashJoin)
		if err != nil {
			log.Warningf("FindLayer: could not disable hash join: %s", err)
		}
		_, err = tx.Exec(disableMergeJoin)
		if err != nil {
			log.Warningf("FindLayer: could not disable merge join: %s", err)
		}

		t = time.Now()
		featureVersions, err := getLayerFeatureVersions(tx, layer.ID)
		observeQueryTime("FindLayer", "getLayerFeatureVersions", t)

		if err != nil {
			return layer, err
		}

		layer.Features = featureVersions

		if withVulnerabilities {
			// Load the vulnerabilities that affect the FeatureVersions.
			t = time.Now()
			err := loadAffectedBy(tx, layer.Features)
			observeQueryTime("FindLayer", "loadAffectedBy", t)

			if err != nil {
				return layer, err
			}
		}
	}

	return layer, nil
}

// Internally, only Feature additions/removals are stored for each layer. If a layer has a parent,
// the Feature list will be compared to the parent's Feature list and the difference will be stored.
// Note that when the Namespace of a layer differs from its parent, it is expected that several
// Feature that were already included a parent will have their Namespace updated as well
// (happens when Feature detectors relies on the detected layer Namespace). However, if the listed
// Feature has the same Name/Version as its parent, InsertLayer considers that the Feature hasn't
// been modified.
func (pgSQL *pgSQL) InsertLayer(layer database.Layer) error {
	tf := time.Now()

	// Verify parameters
	if layer.Name == "" {
		log.Warning("could not insert a layer which has an empty Name")
		return cerrors.NewBadRequestError("could not insert a layer which has an empty Name")
	}

	// Get a potentially existing layer.
	existingLayer, err := pgSQL.FindLayer(layer.Name, true, false)
	if err != nil && err != cerrors.ErrNotFound {
		return err
	} else if err == nil {
		if existingLayer.EngineVersion >= layer.EngineVersion {
			// The layer exists and has an equal or higher engine version, do nothing.
			return nil
		}

		layer.ID = existingLayer.ID
	}

	// We do `defer observeQueryTime` here because we don't want to observe existing layers.
	defer observeQueryTime("InsertLayer", "all", tf)

	// Get parent ID.
	var parentID zero.Int
	if layer.Parent != nil {
		if layer.Parent.ID == 0 {
			log.Warning("Parent is expected to be retrieved from database when inserting a layer.")
			return cerrors.NewBadRequestError("Parent is expected to be retrieved from database when inserting a layer.")
		}

		parentID = zero.IntFrom(int64(layer.Parent.ID))
	}

	// Find or insert namespace if provided.
	var namespaceID zero.Int
	if layer.Namespace != nil {
		n, err := pgSQL.insertNamespace(*layer.Namespace)
		if err != nil {
			return err
		}
		namespaceID = zero.IntFrom(int64(n))
	} else if layer.Namespace == nil && layer.Parent != nil {
		// Import the Namespace from the parent if it has one and this layer doesn't specify one.
		if layer.Parent.Namespace != nil {
			namespaceID = zero.IntFrom(int64(layer.Parent.Namespace.ID))
		}
	}

	// Begin transaction.
	tx, err := pgSQL.Begin()
	if err != nil {
		tx.Rollback()
		return handleError("InsertLayer.Begin()", err)
	}

	if layer.ID == 0 {
		// Insert a new layer.
		err = tx.QueryRow(insertLayer, layer.Name, layer.EngineVersion, parentID, namespaceID).
			Scan(&layer.ID)
		if err != nil {
			tx.Rollback()

			if isErrUniqueViolation(err) {
				// Ignore this error, another process collided.
				log.Debug("Attempted to insert duplicate layer.")
				return nil
			}
			return handleError("insertLayer", err)
		}
	} else {
		// Update an existing layer.
		_, err = tx.Exec(updateLayer, layer.ID, layer.EngineVersion, namespaceID)
		if err != nil {
			tx.Rollback()
			return handleError("updateLayer", err)
		}

		// Remove all existing Layer_diff_FeatureVersion.
		_, err = tx.Exec(removeLayerDiffFeatureVersion, layer.ID)
		if err != nil {
			tx.Rollback()
			return handleError("removeLayerDiffFeatureVersion", err)
		}
	}

	// Update Layer_diff_FeatureVersion now.
	err = pgSQL.updateDiffFeatureVersions(tx, &layer, &existingLayer)
	if err != nil {
		tx.Rollback()
		return err
//.........这里部分代码省略.........

func testInsertLayerUpdate(t *testing.T, datastore database.Datastore) {
	f7 := database.FeatureVersion{
		Feature: database.Feature{
			Namespace: database.Namespace{Name: "TestInsertLayerNamespace3"},
			Name:      "TestInsertLayerFeature7",
		},
		Version: types.NewVersionUnsafe("0.01"),
	}

	l3, _ := datastore.FindLayer("TestInsertLayer3", true, false)
	l3u := database.Layer{
		Name:      l3.Name,
		Parent:    l3.Parent,
		Namespace: &database.Namespace{Name: "TestInsertLayerNamespaceUpdated1"},
		Features:  []database.FeatureVersion{f7},
	}

	l4u := database.Layer{
		Name:          "TestInsertLayer4",
		Parent:        &database.Layer{Name: "TestInsertLayer3"},
		Features:      []database.FeatureVersion{f7},
		EngineVersion: 2,
	}

	// Try to re-insert without increasing the EngineVersion.
	err := datastore.InsertLayer(l3u)
	assert.Nil(t, err)

	l3uf, err := datastore.FindLayer(l3u.Name, true, false)
	if assert.Nil(t, err) {
		assert.Equal(t, l3.Namespace.Name, l3uf.Namespace.Name)
		assert.Equal(t, l3.EngineVersion, l3uf.EngineVersion)
		assert.Len(t, l3uf.Features, len(l3.Features))
	}

	// Update layer l3.
	// Verify that the Namespace, EngineVersion and FeatureVersions got updated.
	l3u.EngineVersion = 2
	err = datastore.InsertLayer(l3u)
	assert.Nil(t, err)

	l3uf, err = datastore.FindLayer(l3u.Name, true, false)
	if assert.Nil(t, err) {
		assert.Equal(t, l3u.Namespace.Name, l3uf.Namespace.Name)
		assert.Equal(t, l3u.EngineVersion, l3uf.EngineVersion)
		if assert.Len(t, l3uf.Features, 1) {
			assert.True(t, cmpFV(l3uf.Features[0], f7), "Updated layer should have %#v but actually have %#v", f7, l3uf.Features[0])
		}
	}

	// Update layer l4.
	// Verify that the Namespace got updated from its new Parent's, and also verify the
	// EnginVersion and FeatureVersions.
	l4u.Parent = &l3uf
	err = datastore.InsertLayer(l4u)
	assert.Nil(t, err)

	l4uf, err := datastore.FindLayer(l3u.Name, true, false)
	if assert.Nil(t, err) {
		assert.Equal(t, l3u.Namespace.Name, l4uf.Namespace.Name)
		assert.Equal(t, l4u.EngineVersion, l4uf.EngineVersion)
		if assert.Len(t, l4uf.Features, 1) {
			assert.True(t, cmpFV(l3uf.Features[0], f7), "Updated layer should have %#v but actually have %#v", f7, l4uf.Features[0])
		}
	}
}