Golang filestorage.NewFileStorageReader函数代码示例

func (*metadataHelperSuite) TestResolveMetadata(c *gc.C) {
	var versionStrings = []string{"1.2.3-precise-amd64"}
	dir := c.MkDir()
	toolstesting.MakeTools(c, dir, "released", versionStrings)
	toolsList := coretools.List{{
		Version: version.MustParseBinary(versionStrings[0]),
		Size:    123,
		SHA256:  "abc",
	}}

	stor, err := filestorage.NewFileStorageReader(dir)
	c.Assert(err, jc.ErrorIsNil)
	err = tools.ResolveMetadata(stor, "released", nil)
	c.Assert(err, jc.ErrorIsNil)

	// We already have size/sha256, so ensure that storage isn't consulted.
	countingStorage := &countingStorage{StorageReader: stor}
	metadata := tools.MetadataFromTools(toolsList, "released")
	err = tools.ResolveMetadata(countingStorage, "released", metadata)
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(countingStorage.counter, gc.Equals, 0)

	// Now clear size/sha256, and check that it is called, and
	// the size/sha256 sum are updated.
	metadata[0].Size = 0
	metadata[0].SHA256 = ""
	err = tools.ResolveMetadata(countingStorage, "released", metadata)
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(countingStorage.counter, gc.Equals, 1)
	c.Assert(metadata[0].Size, gc.Not(gc.Equals), 0)
	c.Assert(metadata[0].SHA256, gc.Not(gc.Equals), "")
}

func (s *filestorageSuite) SetUpTest(c *gc.C) {
	s.dir = c.MkDir()
	var err error
	s.reader, err = filestorage.NewFileStorageReader(s.dir)
	c.Assert(err, jc.ErrorIsNil)
	s.writer, err = filestorage.NewFileStorageWriter(s.dir)
	c.Assert(err, jc.ErrorIsNil)
}

func (s *filestorageSuite) TestPathRelativeToHome(c *gc.C) {
	homeDir := utils.Home()
	tempDir, err := ioutil.TempDir(homeDir, "")
	c.Assert(err, jc.ErrorIsNil)
	defer os.RemoveAll(tempDir)
	dirName := strings.Replace(tempDir, homeDir, "", -1)
	reader, err := filestorage.NewFileStorageReader(filepath.Join(utils.Home(), dirName))
	c.Assert(err, jc.ErrorIsNil)
	url, err := reader.URL("")
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(url, gc.Equals, utils.MakeFileURL(filepath.Join(homeDir, dirName)))
}

func (c *toolsMetadataCommand) Run(context *cmd.Context) error {
	loggo.RegisterWriter("toolsmetadata", cmd.NewCommandLogWriter("juju.environs.tools", context.Stdout, context.Stderr), loggo.INFO)
	defer loggo.RemoveWriter("toolsmetadata")
	if c.metadataDir == "" {
		c.metadataDir = osenv.JujuHome()
	} else {
		c.metadataDir = context.AbsPath(c.metadataDir)
	}

	sourceStorage, err := filestorage.NewFileStorageReader(c.metadataDir)
	if err != nil {
		return err
	}

	// We now store the tools in a directory named after their stream, but the
	// legacy behaviour is to store all tools in a single "releases" directory.
	toolsDir := c.stream
	if c.stream == "" {
		fmt.Fprintf(context.Stdout, "No stream specified, defaulting to released tools in the releases directory.\n")
		c.stream = envtools.ReleasedStream
		toolsDir = envtools.LegacyReleaseDirectory
	}
	fmt.Fprintf(context.Stdout, "Finding tools in %s for stream %s.\n", c.metadataDir, c.stream)
	const minorVersion = -1
	toolsList, err := envtools.ReadList(sourceStorage, toolsDir, version.Current.Major, minorVersion)
	if err == envtools.ErrNoTools {
		var source string
		source, err = envtools.ToolsURL(envtools.DefaultBaseURL)
		if err != nil {
			return err
		}
		sourceDataSource := simplestreams.NewURLDataSource("local source", source, utils.VerifySSLHostnames)
		toolsList, err = envtools.FindToolsForCloud(
			[]simplestreams.DataSource{sourceDataSource}, simplestreams.CloudSpec{}, c.stream,
			version.Current.Major, minorVersion, coretools.Filter{})
	}
	if err != nil {
		return err
	}

	targetStorage, err := filestorage.NewFileStorageWriter(c.metadataDir)
	if err != nil {
		return err
	}
	writeMirrors := envtools.DoNotWriteMirrors
	if c.public {
		writeMirrors = envtools.WriteMirrors
	}
	return mergeAndWriteMetadata(targetStorage, toolsDir, c.stream, c.clean, toolsList, writeMirrors)
}

func (*metadataHelperSuite) TestResolveMetadataLegacyPPC64(c *gc.C) {
	var versionStrings = []string{"1.2.3-precise-amd64", "1.2.3-precise-ppc64el"}
	dir := c.MkDir()
	toolstesting.MakeTools(c, dir, "released", versionStrings)

	toolsList := coretools.List{
		{
			Version: version.MustParseBinary(versionStrings[0]),
		}, {
			Version: version.MustParseBinary(versionStrings[1]),
		}, {
			Version: version.MustParseBinary("1.2.3-precise-ppc64"),
		},
	}
	toolsMetadata := tools.MetadataFromTools(toolsList, dir)
	stor, err := filestorage.NewFileStorageReader(dir)
	c.Assert(err, jc.ErrorIsNil)
	err = tools.ResolveMetadata(stor, "released", toolsMetadata)
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(toolsMetadata, gc.DeepEquals, []*tools.ToolsMetadata{
		{
			Release:  "precise",
			Version:  "1.2.3",
			Arch:     "amd64",
			Size:     19,
			FileType: "tar.gz",
			SHA256:   "dcdd65b962b804a3d63b108d670290ee95a867a97fe9b9f99b2b77b5c7173e59",
			Path:     fmt.Sprintf("%s/juju-1.2.3-precise-amd64.tgz", dir),
		},
		{
			Release:  "precise",
			Version:  "1.2.3",
			Arch:     "ppc64el",
			Size:     21,
			FileType: "tar.gz",
			SHA256:   "a3460ed45eb07a69adfcd541413a495f988c5842d715c6a40353075c3ad47af2",
			Path:     fmt.Sprintf("%s/juju-1.2.3-precise-ppc64el.tgz", dir),
		},
		{
			Release:  "precise",
			Version:  "1.2.3",
			Arch:     "ppc64",
			Size:     21,
			FileType: "tar.gz",
			SHA256:   "a3460ed45eb07a69adfcd541413a495f988c5842d715c6a40353075c3ad47af2",
			Path:     fmt.Sprintf("%s/juju-1.2.3-precise-ppc64el.tgz", dir),
		},
	})
}

func (c *toolsMetadataCommand) Run(context *cmd.Context) error {
	writer := loggo.NewMinimumLevelWriter(
		cmd.NewCommandLogWriter("juju.environs.tools", context.Stdout, context.Stderr),
		loggo.INFO)
	loggo.RegisterWriter("toolsmetadata", writer)
	defer loggo.RemoveWriter("toolsmetadata")
	if c.metadataDir == "" {
		c.metadataDir = osenv.JujuXDGDataHomeDir()
	} else {
		c.metadataDir = context.AbsPath(c.metadataDir)
	}

	sourceStorage, err := filestorage.NewFileStorageReader(c.metadataDir)
	if err != nil {
		return err
	}

	// We now store the tools in a directory named after their stream, but the
	// legacy behaviour is to store all tools in a single "releases" directory.
	toolsDir := c.stream
	if c.stream == "" {
		fmt.Fprintln(context.Stdout, "No stream specified, defaulting to released tools in the releases directory.")
		c.stream = envtools.ReleasedStream
		toolsDir = envtools.LegacyReleaseDirectory
	}
	fmt.Fprintf(context.Stdout, "Finding tools in %s for stream %s.\n", c.metadataDir, c.stream)
	toolsList, err := envtools.ReadList(sourceStorage, toolsDir, -1, -1)
	if err == envtools.ErrNoTools {
		var source string
		source, err = envtools.ToolsURL(envtools.DefaultBaseURL)
		if err != nil {
			return err
		}
		toolsList, err = envtools.FindToolsForCloud(toolsDataSources(source), simplestreams.CloudSpec{}, c.stream, -1, -1, coretools.Filter{})
	}
	if err != nil {
		return err
	}

	targetStorage, err := filestorage.NewFileStorageWriter(c.metadataDir)
	if err != nil {
		return err
	}
	writeMirrors := envtools.DoNotWriteMirrors
	if c.public {
		writeMirrors = envtools.WriteMirrors
	}
	return mergeAndWriteMetadata(targetStorage, toolsDir, c.stream, c.clean, toolsList, writeMirrors)
}

func (s *filestorageSuite) TestRelativePath(c *gc.C) {
	dir := c.MkDir()
	err := os.MkdirAll(filepath.Join(dir, "a", "b", "c"), os.ModePerm)
	c.Assert(err, jc.ErrorIsNil)
	cwd, err := os.Getwd()
	c.Assert(err, jc.ErrorIsNil)
	err = os.Chdir(filepath.Join(dir, "a", "b", "c"))
	c.Assert(err, jc.ErrorIsNil)
	defer os.Chdir(cwd)
	reader, err := filestorage.NewFileStorageReader("../..")
	c.Assert(err, jc.ErrorIsNil)
	url, err := reader.URL("")
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(url, gc.Equals, utils.MakeFileURL(dir)+"/a")
}

// migrateCustomImageMetadata copies uploaded image metadata from provider
// storage to environment storage, preserving paths.
func migrateCustomImageMetadata(st *state.State, agentConfig agent.Config) error {
	logger.Debugf("migrating custom image metadata to environment storage")
	estor := newStateStorage(st.EnvironUUID(), st.MongoSession())

	// Local and manual provider host storage on the state server's
	// filesystem, and serve via HTTP storage. The storage worker
	// doesn't run yet, so we just open the files directly.
	var pstor storage.StorageReader
	providerType := agentConfig.Value(agent.ProviderType)
	if providerType == provider.Local || provider.IsManual(providerType) {
		storageDir := agentConfig.Value(agent.StorageDir)
		var err error
		pstor, err = filestorage.NewFileStorageReader(storageDir)
		if err != nil {
			return errors.Annotate(err, "cannot get local filesystem storage reader")
		}
	} else {
		var err error
		pstor, err = environs.LegacyStorage(st)
		if errors.IsNotSupported(err) {
			return nil
		} else if err != nil {
			return errors.Annotate(err, "cannot get provider storage")
		}
	}

	paths, err := pstor.List(storage.BaseImagesPath)
	if err != nil {
		return err
	}
	for _, path := range paths {
		logger.Infof("migrating image metadata at path %q", path)
		data, err := readImageMetadata(pstor, path)
		if err != nil {
			return errors.Annotate(err, "failed to read image metadata")
		}
		err = estor.Put(path, bytes.NewReader(data), int64(len(data)))
		if err != nil {
			return errors.Annotate(err, "failed to write image metadata")
		}
	}
	return nil
}

func (c *ToolsMetadataCommand) Run(context *cmd.Context) error {
	loggo.RegisterWriter("toolsmetadata", cmd.NewCommandLogWriter("juju.environs.tools", context.Stdout, context.Stderr), loggo.INFO)
	defer loggo.RemoveWriter("toolsmetadata")
	if c.metadataDir == "" {
		c.metadataDir = osenv.JujuHome()
	} else {
		c.metadataDir = context.AbsPath(c.metadataDir)
	}

	sourceStorage, err := filestorage.NewFileStorageReader(c.metadataDir)
	if err != nil {
		return err
	}
	fmt.Fprintf(context.Stdout, "Finding tools in %s\n", c.metadataDir)
	const minorVersion = -1
	toolsList, err := envtools.ReadList(sourceStorage, version.Current.Major, minorVersion)
	if err == envtools.ErrNoTools {
		var source string
		source, err = envtools.ToolsURL(envtools.DefaultBaseURL)
		if err != nil {
			return err
		}
		sourceDataSource := simplestreams.NewURLDataSource("local source", source, utils.VerifySSLHostnames)
		toolsList, err = envtools.FindToolsForCloud(
			[]simplestreams.DataSource{sourceDataSource}, simplestreams.CloudSpec{},
			version.Current.Major, minorVersion, coretools.Filter{})
	}
	if err != nil {
		return err
	}

	targetStorage, err := filestorage.NewFileStorageWriter(c.metadataDir)
	if err != nil {
		return err
	}
	writeMirrors := envtools.DoNotWriteMirrors
	if c.public {
		writeMirrors = envtools.WriteMirrors
	}
	return mergeAndWriteMetadata(targetStorage, toolsList, writeMirrors)
}

func (c *toolsMetadataCommand) Run(context *cmd.Context) error {
	writer := loggo.NewMinimumLevelWriter(
		cmd.NewCommandLogWriter("juju.environs.tools", context.Stdout, context.Stderr),
		loggo.INFO)
	loggo.RegisterWriter("toolsmetadata", writer)
	defer loggo.RemoveWriter("toolsmetadata")
	if c.metadataDir == "" {
		c.metadataDir = osenv.JujuXDGDataHomeDir()
	} else {
		c.metadataDir = context.AbsPath(c.metadataDir)
	}

	sourceStorage, err := filestorage.NewFileStorageReader(c.metadataDir)
	if err != nil {
		return errors.Trace(err)
	}

	fmt.Fprintf(context.Stdout, "Finding tools in %s for stream %s.\n", c.metadataDir, c.stream)
	toolsList, err := envtools.ReadList(sourceStorage, c.stream, -1, -1)
	if err == envtools.ErrNoTools {
		var source string
		source, err = envtools.ToolsURL(envtools.DefaultBaseURL)
		if err != nil {
			return errors.Trace(err)
		}
		toolsList, err = envtools.FindToolsForCloud(toolsDataSources(source), simplestreams.CloudSpec{}, c.stream, -1, -1, coretools.Filter{})
	}
	if err != nil {
		return errors.Trace(err)
	}

	targetStorage, err := filestorage.NewFileStorageWriter(c.metadataDir)
	if err != nil {
		return errors.Trace(err)
	}
	writeMirrors := envtools.DoNotWriteMirrors
	if c.public {
		writeMirrors = envtools.WriteMirrors
	}
	return errors.Trace(mergeAndWriteMetadata(targetStorage, c.stream, c.stream, c.clean, toolsList, writeMirrors))
}

// ParseMetadataFromDir loads ImageMetadata from the specified directory.
func ParseMetadataFromDir(c *gc.C, metadataDir string) []*imagemetadata.ImageMetadata {
	stor, err := filestorage.NewFileStorageReader(metadataDir)
	c.Assert(err, gc.IsNil)
	return ParseMetadataFromStorage(c, stor)
}

// ParseMetadataFromDir loads ToolsMetadata from the specified directory.
func ParseMetadataFromDir(c *gc.C, metadataDir, stream string, expectMirrors bool) []*tools.ToolsMetadata {
	stor, err := filestorage.NewFileStorageReader(metadataDir)
	c.Assert(err, jc.ErrorIsNil)
	return ParseMetadataFromStorage(c, stor, stream, expectMirrors)
}

// migrateToolsStorage copies tools from provider storage to
// environment storage.
func migrateToolsStorage(st *state.State, agentConfig agent.Config) error {
	logger.Debugf("migrating tools to environment storage")

	tstor, err := stateToolsStorage(st)
	if err != nil {
		return errors.Annotate(err, "cannot get tools storage")
	}
	defer tstor.Close()

	// Local and manual provider host storage on the state server's
	// filesystem, and serve via HTTP storage. The storage worker
	// doesn't run yet, so we just open the files directly.
	var stor storage.StorageReader
	providerType := agentConfig.Value(agent.ProviderType)
	if providerType == provider.Local || provider.IsManual(providerType) {
		storageDir := agentConfig.Value(agent.StorageDir)
		var err error
		stor, err = filestorage.NewFileStorageReader(storageDir)
		if err != nil {
			return errors.Annotate(err, "cannot get local filesystem storage reader")
		}
	} else {
		var err error
		stor, err = environs.LegacyStorage(st)
		if errors.IsNotSupported(err) {
			return nil
		} else if err != nil {
			return errors.Annotate(err, "cannot get provider storage")
		}
	}

	// Search provider storage for tools.
	datasource := storage.NewStorageSimpleStreamsDataSource("provider storage", stor, storage.BaseToolsPath)
	toolsList, err := envtools.FindToolsForCloud(
		[]simplestreams.DataSource{datasource},
		simplestreams.CloudSpec{},
		envtools.ReleasedStream,
		-1, -1, tools.Filter{})
	switch err {
	case nil:
		break
	case tools.ErrNoMatches, envtools.ErrNoTools:
		// No tools in provider storage: nothing to do.
		return nil
	default:
		return errors.Annotate(err, "cannot find tools in provider storage")
	}

	for _, agentTools := range toolsList {
		logger.Infof("migrating %v tools to environment storage", agentTools.Version)
		data, err := fetchToolsArchive(stor, envtools.LegacyReleaseDirectory, agentTools)
		if err != nil {
			return errors.Annotatef(err, "failed to fetch %v tools", agentTools.Version)
		}
		err = tstor.AddTools(bytes.NewReader(data), toolstorage.Metadata{
			Version: agentTools.Version,
			Size:    agentTools.Size,
			SHA256:  agentTools.SHA256,
		})
		if err != nil {
			return errors.Annotatef(err, "failed to add %v tools to environment storage", agentTools.Version)
		}
	}
	return nil
}