Golang storage.NewStorageSimpleStreamsDataSource函数代码示例

func (env *mockEnviron) GetToolsSources() ([]simplestreams.DataSource, error) {
	if env.getToolsSources != nil {
		return env.getToolsSources()
	}
	datasource := storage.NewStorageSimpleStreamsDataSource("test cloud storage", env.Storage(), storage.BaseToolsPath)
	return []simplestreams.DataSource{datasource}, nil
}

func (s *bootstrapSuite) setupBootstrapSpecificVersion(c *gc.C, clientMajor, clientMinor int, toolsVersion *version.Number) (error, int, version.Number) {
	currentVersion := version.Current
	currentVersion.Major = clientMajor
	currentVersion.Minor = clientMinor
	currentVersion.Tag = ""
	s.PatchValue(&version.Current, currentVersion)
	s.PatchValue(&series.HostSeries, func() string { return "trusty" })
	s.PatchValue(&arch.HostArch, func() string { return arch.AMD64 })

	env := newEnviron("foo", useDefaultKeys, nil)
	s.setDummyStorage(c, env)
	envtools.RegisterToolsDataSourceFunc("local storage", func(environs.Environ) (simplestreams.DataSource, error) {
		return storage.NewStorageSimpleStreamsDataSource("test datasource", env.storage, "tools", simplestreams.CUSTOM_CLOUD_DATA, false), nil
	})
	defer envtools.UnregisterToolsDataSourceFunc("local storage")

	toolsBinaries := []version.Binary{
		version.MustParseBinary("10.11.12-trusty-amd64"),
		version.MustParseBinary("10.11.13-trusty-amd64"),
		version.MustParseBinary("10.11-beta1-trusty-amd64"),
	}
	stream := "released"
	if toolsVersion != nil && toolsVersion.Tag != "" {
		stream = "devel"
		currentVersion.Tag = toolsVersion.Tag
	}
	_, err := envtesting.UploadFakeToolsVersions(env.storage, stream, stream, toolsBinaries...)
	c.Assert(err, jc.ErrorIsNil)

	err = bootstrap.Bootstrap(envtesting.BootstrapContext(c), env, bootstrap.BootstrapParams{
		AgentVersion: toolsVersion,
	})
	vers, _ := env.cfg.AgentVersion()
	return err, env.bootstrapCount, vers
}

func (s *bootstrapSuite) setupBootstrapSpecificVersion(
	c *gc.C, clientMajor, clientMinor int, toolsVersion *version.Number,
) (error, int, version.Number) {
	currentVersion := version.Current
	currentVersion.Major = clientMajor
	currentVersion.Minor = clientMinor
	currentVersion.Series = "trusty"
	currentVersion.Arch = "amd64"
	s.PatchValue(&version.Current, currentVersion)

	env := newEnviron("foo", useDefaultKeys, nil)
	s.setDummyStorage(c, env)
	envtools.RegisterToolsDataSourceFunc("local storage", func(environs.Environ) (simplestreams.DataSource, error) {
		return storage.NewStorageSimpleStreamsDataSource("test datasource", env.storage, "tools"), nil
	})
	defer envtools.UnregisterToolsDataSourceFunc("local storage")

	toolsBinaries := []version.Binary{
		version.MustParseBinary("10.11.12-trusty-amd64"),
		version.MustParseBinary("10.11.13-trusty-amd64"),
	}
	_, err := envtesting.UploadFakeToolsVersions(env.storage, "released", "released", toolsBinaries...)
	c.Assert(err, jc.ErrorIsNil)

	err = bootstrap.Bootstrap(envtesting.BootstrapContext(c), env, bootstrap.BootstrapParams{
		AgentVersion: toolsVersion,
	})
	vers, _ := env.cfg.AgentVersion()
	return err, env.bootstrapCount, vers
}

func (s *datasourceSuite) TestURLWithBasePath(c *gc.C) {
	sampleData := "hello world"
	s.stor.Put("base/bar/data.txt", bytes.NewReader([]byte(sampleData)), int64(len(sampleData)))
	ds := storage.NewStorageSimpleStreamsDataSource("test datasource", s.stor, "base")
	url, err := ds.URL("bar")
	c.Assert(err, gc.IsNil)
	expectedURL, _ := s.stor.URL("base/bar")
	c.Assert(url, gc.Equals, expectedURL)
}

// ParseMetadataFromStorage loads ImageMetadata from the specified storage reader.
func ParseMetadataFromStorage(c *gc.C, stor storage.StorageReader) []*imagemetadata.ImageMetadata {
	source := storage.NewStorageSimpleStreamsDataSource("test storage reader", stor, "images")

	// Find the simplestreams index file.
	params := simplestreams.ValueParams{
		DataType:      "image-ids",
		ValueTemplate: imagemetadata.ImageMetadata{},
	}
	const requireSigned = false
	indexPath := simplestreams.UnsignedIndex
	indexRef, err := simplestreams.GetIndexWithFormat(
		source, indexPath, "index:1.0", requireSigned, simplestreams.CloudSpec{}, params)
	c.Assert(err, gc.IsNil)
	c.Assert(indexRef.Indexes, gc.HasLen, 1)

	imageIndexMetadata := indexRef.Indexes["com.ubuntu.cloud:custom"]
	c.Assert(imageIndexMetadata, gc.NotNil)

	// Read the products file contents.
	r, err := stor.Get(path.Join("images", imageIndexMetadata.ProductsFilePath))
	defer r.Close()
	c.Assert(err, gc.IsNil)
	data, err := ioutil.ReadAll(r)
	c.Assert(err, gc.IsNil)

	// Parse the products file metadata.
	url, err := source.URL(imageIndexMetadata.ProductsFilePath)
	c.Assert(err, gc.IsNil)
	cloudMetadata, err := simplestreams.ParseCloudMetadata(data, "products:1.0", url, imagemetadata.ImageMetadata{})
	c.Assert(err, gc.IsNil)

	// Collate the metadata.
	imageMetadataMap := make(map[string]*imagemetadata.ImageMetadata)
	var expectedProductIds set.Strings
	var imageVersions set.Strings
	for _, mc := range cloudMetadata.Products {
		for _, items := range mc.Items {
			for key, item := range items.Items {
				imageMetadata := item.(*imagemetadata.ImageMetadata)
				imageMetadataMap[key] = imageMetadata
				imageVersions.Add(key)
				productId := fmt.Sprintf("com.ubuntu.cloud:server:%s:%s", mc.Version, imageMetadata.Arch)
				expectedProductIds.Add(productId)
			}
		}
	}

	// Make sure index's product IDs are all represented in the products metadata.
	sort.Strings(imageIndexMetadata.ProductIds)
	c.Assert(imageIndexMetadata.ProductIds, gc.DeepEquals, expectedProductIds.SortedValues())

	imageMetadata := make([]*imagemetadata.ImageMetadata, len(imageMetadataMap))
	for i, key := range imageVersions.SortedValues() {
		imageMetadata[i] = imageMetadataMap[key]
	}
	return imageMetadata
}

func (s *datasourceSuite) TestFetchWithRetry(c *gc.C) {
	stor := &fakeStorage{shouldRetry: true}
	ds := storage.NewStorageSimpleStreamsDataSource("test datasource", stor, "base", simplestreams.DEFAULT_CLOUD_DATA, false)
	ds.SetAllowRetry(true)
	_, _, err := ds.Fetch("foo/bar/data.txt")
	c.Assert(err, gc.ErrorMatches, "an error")
	c.Assert(stor.getName, gc.Equals, "base/foo/bar/data.txt")
	c.Assert(stor.invokeCount, gc.Equals, 10)
}

func (s *datasourceSuite) TestURL(c *gc.C) {
	sampleData := "hello world"
	s.stor.Put("bar/data.txt", bytes.NewReader([]byte(sampleData)), int64(len(sampleData)))
	ds := storage.NewStorageSimpleStreamsDataSource("test datasource", s.stor, "", simplestreams.DEFAULT_CLOUD_DATA, false)
	url, err := ds.URL("bar")
	c.Assert(err, jc.ErrorIsNil)
	expectedURL, _ := s.stor.URL("bar")
	c.Assert(url, gc.Equals, expectedURL)
}

func (s *datasourceSuite) TestFetchWithNoRetry(c *gc.C) {
	// NB shouldRetry below is true indicating the fake storage is capable of
	// retrying, not that it will retry.
	stor := &fakeStorage{shouldRetry: true}
	ds := storage.NewStorageSimpleStreamsDataSource("test datasource", stor, "base", simplestreams.DEFAULT_CLOUD_DATA, false)
	_, _, err := ds.Fetch("foo/bar/data.txt")
	c.Assert(err, gc.ErrorMatches, "an error")
	c.Assert(stor.getName, gc.Equals, "base/foo/bar/data.txt")
	c.Assert(stor.invokeCount, gc.Equals, 1)
}

// readMetadata reads the image metadata from metadataStore.
func readMetadata(metadataStore storage.Storage) ([]*ImageMetadata, error) {
	// Read any existing metadata so we can merge the new tools metadata with what's there.
	dataSource := storage.NewStorageSimpleStreamsDataSource("existing metadata", metadataStore, storage.BaseImagesPath)
	imageConstraint := NewImageConstraint(simplestreams.LookupParams{})
	existingMetadata, _, err := Fetch([]simplestreams.DataSource{dataSource}, imageConstraint, false)
	if err != nil && !errors.IsNotFound(err) {
		return nil, err
	}
	return existingMetadata, nil
}

func (s *datasourceSuite) TestFetchWithBasePath(c *gc.C) {
	sampleData := "hello world"
	s.stor.Put("base/foo/bar/data.txt", bytes.NewReader([]byte(sampleData)), int64(len(sampleData)))
	ds := storage.NewStorageSimpleStreamsDataSource("test datasource", s.stor, "base", simplestreams.DEFAULT_CLOUD_DATA, false)
	rc, url, err := ds.Fetch("foo/bar/data.txt")
	c.Assert(err, jc.ErrorIsNil)
	defer rc.Close()
	c.Assert(url, gc.Equals, s.baseURL+"/base/foo/bar/data.txt")
	data, err := ioutil.ReadAll(rc)
	c.Assert(data, gc.DeepEquals, []byte(sampleData))
}

// ReadMetadata returns the tools metadata from the given storage for the specified stream.
func ReadMetadata(store storage.StorageReader, stream string) ([]*ToolsMetadata, error) {
	dataSource := storage.NewStorageSimpleStreamsDataSource("existing metadata", store, storage.BaseToolsPath, simplestreams.EXISTING_CLOUD_DATA, false)
	toolsConstraint, err := makeToolsConstraint(simplestreams.CloudSpec{}, stream, -1, -1, coretools.Filter{})
	if err != nil {
		return nil, err
	}
	metadata, _, err := Fetch([]simplestreams.DataSource{dataSource}, toolsConstraint)
	if err != nil && !errors.IsNotFound(err) {
		return nil, err
	}
	return metadata, nil
}

func assertFetch(c *gc.C, stor storage.Storage, series, arch, region, endpoint, id string) {
	cons := imagemetadata.NewImageConstraint(simplestreams.LookupParams{
		CloudSpec: simplestreams.CloudSpec{region, endpoint},
		Series:    []string{series},
		Arches:    []string{arch},
	})
	dataSource := storage.NewStorageSimpleStreamsDataSource("test datasource", stor, "images")
	metadata, _, err := imagemetadata.Fetch([]simplestreams.DataSource{dataSource}, cons, false)
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(metadata, gc.HasLen, 1)
	c.Assert(metadata[0].Id, gc.Equals, id)
}

func assertFetch(c *gc.C, stor storage.Storage, series, arch, region, endpoint string, ids ...string) {
	cons := imagemetadata.NewImageConstraint(simplestreams.LookupParams{
		CloudSpec: simplestreams.CloudSpec{region, endpoint},
		Series:    []string{series},
		Arches:    []string{arch},
	})
	dataSource := storage.NewStorageSimpleStreamsDataSource("test datasource", stor, "images", simplestreams.DEFAULT_CLOUD_DATA, false)
	metadata, _, err := imagemetadata.Fetch([]simplestreams.DataSource{dataSource}, cons)
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(metadata, gc.HasLen, len(ids))
	for i, id := range ids {
		c.Assert(metadata[i].Id, gc.Equals, id)
	}
}

func (s *bootstrapSuite) setupBootstrapSpecificVersion(c *gc.C, clientMajor, clientMinor int, toolsVersion *version.Number) (error, int, version.Number) {
	currentVersion := jujuversion.Current
	currentVersion.Major = clientMajor
	currentVersion.Minor = clientMinor
	currentVersion.Tag = ""
	s.PatchValue(&jujuversion.Current, currentVersion)
	s.PatchValue(&series.HostSeries, func() string { return "trusty" })
	s.PatchValue(&arch.HostArch, func() string { return arch.AMD64 })

	env := newEnviron("foo", useDefaultKeys, nil)
	s.setDummyStorage(c, env)
	envtools.RegisterToolsDataSourceFunc("local storage", func(environs.Environ) (simplestreams.DataSource, error) {
		return storage.NewStorageSimpleStreamsDataSource("test datasource", env.storage, "tools", simplestreams.CUSTOM_CLOUD_DATA, false), nil
	})
	defer envtools.UnregisterToolsDataSourceFunc("local storage")

	toolsBinaries := []version.Binary{
		version.MustParseBinary("10.11.12-trusty-amd64"),
		version.MustParseBinary("10.11.13-trusty-amd64"),
		version.MustParseBinary("10.11-beta1-trusty-amd64"),
	}
	stream := "released"
	if toolsVersion != nil && toolsVersion.Tag != "" {
		stream = "devel"
		currentVersion.Tag = toolsVersion.Tag
	}
	_, err := envtesting.UploadFakeToolsVersions(env.storage, stream, stream, toolsBinaries...)
	c.Assert(err, jc.ErrorIsNil)

	err = bootstrap.Bootstrap(envtesting.BootstrapContext(c), env, bootstrap.BootstrapParams{
		ControllerConfig: coretesting.FakeControllerConfig(),
		AdminSecret:      "admin-secret",
		CAPrivateKey:     coretesting.CAKey,
		AgentVersion:     toolsVersion,
		BuildAgentTarball: func(build bool, ver *version.Number, _ string) (*sync.BuiltAgent, error) {
			c.Logf("BuildAgentTarball version %s", ver)
			c.Assert(build, jc.IsFalse)
			return &sync.BuiltAgent{Dir: c.MkDir()}, nil
		},
	})
	vers, _ := env.cfg.AgentVersion()
	return err, env.bootstrapCount, vers
}

// ParseIndexMetadataFromStorage loads Indices from the specified storage reader.
func ParseIndexMetadataFromStorage(c *gc.C, stor storage.StorageReader) (*simplestreams.IndexMetadata, simplestreams.DataSource) {
	source := storage.NewStorageSimpleStreamsDataSource("test storage reader", stor, "images", simplestreams.DEFAULT_CLOUD_DATA, false)

	// Find the simplestreams index file.
	params := simplestreams.ValueParams{
		DataType:      "image-ids",
		ValueTemplate: imagemetadata.ImageMetadata{},
	}
	const requireSigned = false
	indexPath := simplestreams.UnsignedIndex("v1", 1)
	mirrorsPath := simplestreams.MirrorsPath("v1")
	indexRef, err := simplestreams.GetIndexWithFormat(
		source, indexPath, "index:1.0", mirrorsPath, requireSigned, simplestreams.CloudSpec{}, params)
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(indexRef.Indexes, gc.HasLen, 1)

	imageIndexMetadata := indexRef.Indexes["com.ubuntu.cloud:custom"]
	c.Assert(imageIndexMetadata, gc.NotNil)
	return imageIndexMetadata, source
}