Golang tools.ReadList函数代码示例

// UploadFakeToolsVersions puts fake tools in the supplied storage for the supplied versions.
func UploadFakeToolsVersions(stor storage.Storage, toolsDir, stream string, versions ...version.Binary) ([]*coretools.Tools, error) {
	// Leave existing tools alone.
	existingTools := make(map[version.Binary]*coretools.Tools)
	existing, _ := envtools.ReadList(stor, toolsDir, 1, -1)
	for _, tools := range existing {
		existingTools[tools.Version] = tools
	}
	var agentTools coretools.List = make(coretools.List, len(versions))
	for i, version := range versions {
		if tools, ok := existingTools[version]; ok {
			agentTools[i] = tools
		} else {
			t, err := uploadFakeToolsVersion(stor, toolsDir, version)
			if err != nil {
				return nil, err
			}
			agentTools[i] = t
		}
	}
	if err := envtools.MergeAndWriteMetadata(stor, toolsDir, stream, agentTools, envtools.DoNotWriteMirrors); err != nil {
		return nil, err
	}
	err := SignTestTools(stor)
	if err != nil {
		return nil, err
	}
	return agentTools, nil
}

func assertNoUnexpectedTools(c *gc.C, stor storage.StorageReader) {
	// We only expect v1.x tools, no v2.x tools.
	list, err := envtools.ReadList(stor, 2, 0)
	if len(list) > 0 {
		c.Logf("got unexpected tools: %s", list)
	}
	c.Assert(err, gc.Equals, coretools.ErrNoMatches)
}

func (c *toolsMetadataCommand) Run(context *cmd.Context) error {
	loggo.RegisterWriter("toolsmetadata", cmd.NewCommandLogWriter("juju.environs.tools", context.Stdout, context.Stderr), loggo.INFO)
	defer loggo.RemoveWriter("toolsmetadata")
	if c.metadataDir == "" {
		c.metadataDir = osenv.JujuHome()
	} else {
		c.metadataDir = context.AbsPath(c.metadataDir)
	}

	sourceStorage, err := filestorage.NewFileStorageReader(c.metadataDir)
	if err != nil {
		return err
	}

	// We now store the tools in a directory named after their stream, but the
	// legacy behaviour is to store all tools in a single "releases" directory.
	toolsDir := c.stream
	if c.stream == "" {
		fmt.Fprintf(context.Stdout, "No stream specified, defaulting to released tools in the releases directory.\n")
		c.stream = envtools.ReleasedStream
		toolsDir = envtools.LegacyReleaseDirectory
	}
	fmt.Fprintf(context.Stdout, "Finding tools in %s for stream %s.\n", c.metadataDir, c.stream)
	const minorVersion = -1
	toolsList, err := envtools.ReadList(sourceStorage, toolsDir, version.Current.Major, minorVersion)
	if err == envtools.ErrNoTools {
		var source string
		source, err = envtools.ToolsURL(envtools.DefaultBaseURL)
		if err != nil {
			return err
		}
		sourceDataSource := simplestreams.NewURLDataSource("local source", source, utils.VerifySSLHostnames)
		toolsList, err = envtools.FindToolsForCloud(
			[]simplestreams.DataSource{sourceDataSource}, simplestreams.CloudSpec{}, c.stream,
			version.Current.Major, minorVersion, coretools.Filter{})
	}
	if err != nil {
		return err
	}

	targetStorage, err := filestorage.NewFileStorageWriter(c.metadataDir)
	if err != nil {
		return err
	}
	writeMirrors := envtools.DoNotWriteMirrors
	if c.public {
		writeMirrors = envtools.WriteMirrors
	}
	return mergeAndWriteMetadata(targetStorage, toolsDir, c.stream, c.clean, toolsList, writeMirrors)
}

func (c *toolsMetadataCommand) Run(context *cmd.Context) error {
	writer := loggo.NewMinimumLevelWriter(
		cmd.NewCommandLogWriter("juju.environs.tools", context.Stdout, context.Stderr),
		loggo.INFO)
	loggo.RegisterWriter("toolsmetadata", writer)
	defer loggo.RemoveWriter("toolsmetadata")
	if c.metadataDir == "" {
		c.metadataDir = osenv.JujuXDGDataHomeDir()
	} else {
		c.metadataDir = context.AbsPath(c.metadataDir)
	}

	sourceStorage, err := filestorage.NewFileStorageReader(c.metadataDir)
	if err != nil {
		return err
	}

	// We now store the tools in a directory named after their stream, but the
	// legacy behaviour is to store all tools in a single "releases" directory.
	toolsDir := c.stream
	if c.stream == "" {
		fmt.Fprintln(context.Stdout, "No stream specified, defaulting to released tools in the releases directory.")
		c.stream = envtools.ReleasedStream
		toolsDir = envtools.LegacyReleaseDirectory
	}
	fmt.Fprintf(context.Stdout, "Finding tools in %s for stream %s.\n", c.metadataDir, c.stream)
	toolsList, err := envtools.ReadList(sourceStorage, toolsDir, -1, -1)
	if err == envtools.ErrNoTools {
		var source string
		source, err = envtools.ToolsURL(envtools.DefaultBaseURL)
		if err != nil {
			return err
		}
		toolsList, err = envtools.FindToolsForCloud(toolsDataSources(source), simplestreams.CloudSpec{}, c.stream, -1, -1, coretools.Filter{})
	}
	if err != nil {
		return err
	}

	targetStorage, err := filestorage.NewFileStorageWriter(c.metadataDir)
	if err != nil {
		return err
	}
	writeMirrors := envtools.DoNotWriteMirrors
	if c.public {
		writeMirrors = envtools.WriteMirrors
	}
	return mergeAndWriteMetadata(targetStorage, toolsDir, c.stream, c.clean, toolsList, writeMirrors)
}

func (s *StorageSuite) TestReadList(c *gc.C) {
	stor, err := filestorage.NewFileStorageWriter(c.MkDir())
	c.Assert(err, jc.ErrorIsNil)
	v100 := version.MustParseBinary("1.0.0-precise-amd64")
	v101 := version.MustParseBinary("1.0.1-precise-amd64")
	v111 := version.MustParseBinary("1.1.1-precise-amd64")
	v201 := version.MustParseBinary("2.0.1-precise-amd64")
	agentTools := envtesting.AssertUploadFakeToolsVersions(c, stor, "proposed", "proposed", v100, v101, v111, v201)
	t100 := agentTools[0]
	t101 := agentTools[1]
	t111 := agentTools[2]
	t201 := agentTools[3]

	for i, t := range []struct {
		majorVersion,
		minorVersion int
		list coretools.List
	}{{
		-1, -1, coretools.List{t100, t101, t111, t201},
	}, {
		1, 0, coretools.List{t100, t101},
	}, {
		1, 1, coretools.List{t111},
	}, {
		1, -1, coretools.List{t100, t101, t111},
	}, {
		1, 2, nil,
	}, {
		3, 0, nil,
	}} {
		c.Logf("test %d", i)
		list, err := envtools.ReadList(stor, "proposed", t.majorVersion, t.minorVersion)
		if t.list != nil {
			c.Assert(err, jc.ErrorIsNil)
			// ReadList doesn't set the Size or SHA256, so blank out those attributes.
			for _, tool := range t.list {
				tool.Size = 0
				tool.SHA256 = ""
			}
			c.Assert(list, gc.DeepEquals, t.list)
		} else {
			c.Assert(err, gc.Equals, coretools.ErrNoMatches)
		}
	}
}

func (s *StorageSuite) TestReadList(c *gc.C) {
	store := s.env.Storage()
	v001 := version.MustParseBinary("0.0.1-precise-amd64")
	v100 := version.MustParseBinary("1.0.0-precise-amd64")
	v101 := version.MustParseBinary("1.0.1-precise-amd64")
	v111 := version.MustParseBinary("1.1.1-precise-amd64")
	agentTools := envtesting.AssertUploadFakeToolsVersions(c, store, v001, v100, v101, v111)
	t001 := agentTools[0]
	t100 := agentTools[1]
	t101 := agentTools[2]
	t111 := agentTools[3]

	for i, t := range []struct {
		majorVersion,
		minorVersion int
		list coretools.List
	}{{
		0, 0, coretools.List{t001},
	}, {
		1, 0, coretools.List{t100, t101},
	}, {
		1, 1, coretools.List{t111},
	}, {
		1, -1, coretools.List{t100, t101, t111},
	}, {
		1, 2, nil,
	}, {
		2, 0, nil,
	}} {
		c.Logf("test %d", i)
		list, err := envtools.ReadList(store, t.majorVersion, t.minorVersion)
		if t.list != nil {
			c.Assert(err, gc.IsNil)
			// ReadList doesn't set the Size of SHA256, so blank out those attributes.
			for _, tool := range t.list {
				tool.Size = 0
				tool.SHA256 = ""
			}
			c.Assert(list, gc.DeepEquals, t.list)
		} else {
			c.Assert(err, gc.Equals, coretools.ErrNoMatches)
		}
	}
}

func (s *uploadSuite) assertUploadedTools(c *gc.C, t *coretools.Tools, expectSeries []string, stream string) {
	s.assertEqualsCurrentVersion(c, t.Version)
	expectRaw := downloadToolsRaw(c, t)

	list, err := envtools.ReadList(s.targetStorage, stream, jujuversion.Current.Major, jujuversion.Current.Minor)
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(list.AllSeries(), jc.SameContents, expectSeries)
	sort.Strings(expectSeries)
	c.Assert(list.AllSeries(), gc.DeepEquals, expectSeries)
	for _, t := range list {
		c.Logf("checking %s", t.URL)
		c.Assert(t.Version.Number, gc.Equals, jujuversion.Current)
		actualRaw := downloadToolsRaw(c, t)
		c.Assert(string(actualRaw), gc.Equals, string(expectRaw))
	}
	metadata, err := envtools.ReadMetadata(s.targetStorage, stream)
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(metadata, gc.HasLen, 0)
}

func (s *uploadSuite) assertUploadedTools(c *gc.C, t *coretools.Tools, uploadedSeries string) {
	c.Assert(t.Version, gc.Equals, version.Current)
	expectRaw := downloadToolsRaw(c, t)

	list, err := envtools.ReadList(s.env.Storage(), version.Current.Major, version.Current.Minor)
	c.Assert(err, gc.IsNil)
	c.Assert(list, gc.HasLen, 3)
	expectSeries := []string{"quantal", uploadedSeries, version.Current.Series}
	sort.Strings(expectSeries)
	c.Assert(list.AllSeries(), gc.DeepEquals, expectSeries)
	for _, t := range list {
		c.Logf("checking %s", t.URL)
		c.Assert(t.Version.Number, gc.Equals, version.Current.Number)
		actualRaw := downloadToolsRaw(c, t)
		c.Assert(string(actualRaw), gc.Equals, string(expectRaw))
	}
	metadata, err := envtools.ReadMetadata(s.env.Storage())
	c.Assert(err, gc.IsNil)
	c.Assert(metadata, gc.HasLen, 0)
}

func (c *ToolsMetadataCommand) Run(context *cmd.Context) error {
	loggo.RegisterWriter("toolsmetadata", cmd.NewCommandLogWriter("juju.environs.tools", context.Stdout, context.Stderr), loggo.INFO)
	defer loggo.RemoveWriter("toolsmetadata")
	if c.metadataDir == "" {
		c.metadataDir = osenv.JujuHome()
	} else {
		c.metadataDir = context.AbsPath(c.metadataDir)
	}

	sourceStorage, err := filestorage.NewFileStorageReader(c.metadataDir)
	if err != nil {
		return err
	}
	fmt.Fprintf(context.Stdout, "Finding tools in %s\n", c.metadataDir)
	const minorVersion = -1
	toolsList, err := envtools.ReadList(sourceStorage, version.Current.Major, minorVersion)
	if err == envtools.ErrNoTools {
		var source string
		source, err = envtools.ToolsURL(envtools.DefaultBaseURL)
		if err != nil {
			return err
		}
		sourceDataSource := simplestreams.NewURLDataSource("local source", source, utils.VerifySSLHostnames)
		toolsList, err = envtools.FindToolsForCloud(
			[]simplestreams.DataSource{sourceDataSource}, simplestreams.CloudSpec{},
			version.Current.Major, minorVersion, coretools.Filter{})
	}
	if err != nil {
		return err
	}

	targetStorage, err := filestorage.NewFileStorageWriter(c.metadataDir)
	if err != nil {
		return err
	}
	writeMirrors := envtools.DoNotWriteMirrors
	if c.public {
		writeMirrors = envtools.WriteMirrors
	}
	return mergeAndWriteMetadata(targetStorage, toolsList, writeMirrors)
}

func (c *toolsMetadataCommand) Run(context *cmd.Context) error {
	writer := loggo.NewMinimumLevelWriter(
		cmd.NewCommandLogWriter("juju.environs.tools", context.Stdout, context.Stderr),
		loggo.INFO)
	loggo.RegisterWriter("toolsmetadata", writer)
	defer loggo.RemoveWriter("toolsmetadata")
	if c.metadataDir == "" {
		c.metadataDir = osenv.JujuXDGDataHomeDir()
	} else {
		c.metadataDir = context.AbsPath(c.metadataDir)
	}

	sourceStorage, err := filestorage.NewFileStorageReader(c.metadataDir)
	if err != nil {
		return errors.Trace(err)
	}

	fmt.Fprintf(context.Stdout, "Finding tools in %s for stream %s.\n", c.metadataDir, c.stream)
	toolsList, err := envtools.ReadList(sourceStorage, c.stream, -1, -1)
	if err == envtools.ErrNoTools {
		var source string
		source, err = envtools.ToolsURL(envtools.DefaultBaseURL)
		if err != nil {
			return errors.Trace(err)
		}
		toolsList, err = envtools.FindToolsForCloud(toolsDataSources(source), simplestreams.CloudSpec{}, c.stream, -1, -1, coretools.Filter{})
	}
	if err != nil {
		return errors.Trace(err)
	}

	targetStorage, err := filestorage.NewFileStorageWriter(c.metadataDir)
	if err != nil {
		return errors.Trace(err)
	}
	writeMirrors := envtools.DoNotWriteMirrors
	if c.public {
		writeMirrors = envtools.WriteMirrors
	}
	return errors.Trace(mergeAndWriteMetadata(targetStorage, c.stream, c.stream, c.clean, toolsList, writeMirrors))
}

func (s *StorageSuite) TestReadListLegacyPPC64(c *gc.C) {
	stor, err := filestorage.NewFileStorageWriter(c.MkDir())
	c.Assert(err, jc.ErrorIsNil)
	v100 := version.MustParseBinary("1.0.0-precise-amd64")
	v101 := version.MustParseBinary("1.0.1-precise-ppc64el")
	agentTools := envtesting.AssertUploadFakeToolsVersions(c, stor, "proposed", "proposed", v100, v101)

	amd64Tools := agentTools[0]
	ppc64elTools := agentTools[1]
	// We also expect metadata for ppc64 to be added.
	ppc64Tools := *ppc64elTools
	ppc64Tools.Version.Arch = "ppc64"
	expected := coretools.List{amd64Tools, ppc64elTools, &ppc64Tools}

	list, err := envtools.ReadList(stor, "proposed", 1, 0)
	c.Assert(err, jc.ErrorIsNil)
	// ReadList doesn't set the Size or SHA256, so blank out those attributes.
	for _, tool := range expected {
		tool.Size = 0
		tool.SHA256 = ""
	}
	c.Assert(list, gc.DeepEquals, expected)
}

func (s *StorageSuite) TestReadListEmpty(c *gc.C) {
	stor, err := filestorage.NewFileStorageWriter(c.MkDir())
	c.Assert(err, jc.ErrorIsNil)
	_, err = envtools.ReadList(stor, "released", 2, 0)
	c.Assert(err, gc.Equals, envtools.ErrNoTools)
}

// SyncTools copies the Juju tools tarball from the official bucket
// or a specified source directory into the user's environment.
func SyncTools(syncContext *SyncContext) error {
	sourceDataSource, err := selectSourceDatasource(syncContext)
	if err != nil {
		return err
	}

	logger.Infof("listing available tools")
	if syncContext.MajorVersion == 0 && syncContext.MinorVersion == 0 {
		syncContext.MajorVersion = version.Current.Major
		syncContext.MinorVersion = -1
		if !syncContext.AllVersions {
			syncContext.MinorVersion = version.Current.Minor
		}
	} else if !syncContext.Dev && syncContext.MinorVersion != -1 {
		// If a major.minor version is specified, we allow dev versions.
		// If Dev is already true, leave it alone.
		syncContext.Dev = true
	}

	released := !syncContext.Dev && !version.Current.IsDev()
	sourceTools, err := envtools.FindToolsForCloud(
		[]simplestreams.DataSource{sourceDataSource}, simplestreams.CloudSpec{},
		syncContext.MajorVersion, syncContext.MinorVersion, coretools.Filter{Released: released})
	if err != nil {
		return err
	}

	logger.Infof("found %d tools", len(sourceTools))
	if !syncContext.AllVersions {
		var latest version.Number
		latest, sourceTools = sourceTools.Newest()
		logger.Infof("found %d recent tools (version %s)", len(sourceTools), latest)
	}
	for _, tool := range sourceTools {
		logger.Debugf("found source tool: %v", tool)
	}

	logger.Infof("listing target tools storage")
	targetStorage := syncContext.Target
	targetTools, err := envtools.ReadList(targetStorage, syncContext.MajorVersion, -1)
	switch err {
	case nil, coretools.ErrNoMatches, envtools.ErrNoTools:
	default:
		return err
	}
	for _, tool := range targetTools {
		logger.Debugf("found target tool: %v", tool)
	}

	missing := sourceTools.Exclude(targetTools)
	logger.Infof("found %d tools in target; %d tools to be copied", len(targetTools), len(missing))
	err = copyTools(missing, syncContext, targetStorage)
	if err != nil {
		return err
	}
	logger.Infof("copied %d tools", len(missing))

	logger.Infof("generating tools metadata")
	if !syncContext.DryRun {
		targetTools = append(targetTools, missing...)
		writeMirrors := envtools.DoNotWriteMirrors
		if syncContext.Public {
			writeMirrors = envtools.WriteMirrors
		}
		err = envtools.MergeAndWriteMetadata(targetStorage, targetTools, writeMirrors)
		if err != nil {
			return err
		}
	}
	logger.Infof("tools metadata written")
	return nil
}

func (f StorageToolsFinder) FindTools(major int) (coretools.List, error) {
	return envtools.ReadList(f.Storage, major, -1)
}

func (s *StorageSuite) TestReadListEmpty(c *gc.C) {
	store := s.env.Storage()
	_, err := envtools.ReadList(store, 2, 0)
	c.Assert(err, gc.Equals, envtools.ErrNoTools)
}