Golang log4go.Info函数代码示例

// RemoveImages removes least recently pulled docker images from the local docker host.
func RemoveImages(PulledImages []ImageMetadataInfo, imageToMDMap map[string][]ImageMetadataInfo) {
	numRemoved := 0
	for _, imageMD := range PulledImages {
		// Get all metadata (repo/tags) associated with that image
		for _, metadata := range imageToMDMap[imageMD.Image] {
			// basespec := RegistrySpec + "/" + string(t.Repo) + ":"
			if ExcludeRepo[RepoType(metadata.Repo)] {
				continue
			}
			blog.Debug("Removing the following registry/repo:tag: " + RegistrySpec + "/" +
				metadata.Repo + ":" + metadata.Tag)
			apipath := "/images/" + RegistrySpec + "/" + metadata.Repo + ":" + metadata.Tag
			blog.Info("RemoveImages %s", apipath)
			config.BanyanUpdate("Remove", apipath)
			_, err := DockerAPI(DockerTransport, "DELETE", apipath, []byte{}, "")
			if err != nil {
				blog.Error(err, "RemoveImages Repo:Tag", metadata.Repo, metadata.Tag,
					"image", metadata.Image)
			}
			numRemoved++
		}
	}

	blog.Info("Number of repo/tags removed this time around: %d", numRemoved)
	return
}

// GetNewImageMetadata takes the set of existing images, queries the registry to find any changes,
// and then brings the Output Writer up to date by telling it the obsolete metadata to delete
// and the new metadata to add.
func GetNewImageMetadata(oldMetadataSet MetadataSet) (tagSlice []TagInfo,
	metadataSlice []ImageMetadataInfo, currentMetadataSet MetadataSet) {

	var currentMetadataSlice []ImageMetadataInfo
	//config.BanyanUpdate("Loading Registry Metadata")
	if LocalHost == true {
		blog.Info("Collect images from local Docker host")
		currentMetadataSlice = GetLocalImageMetadata(oldMetadataSet)
		// there is no tag API under Docker Remote API,
		// and the caller of GetNewImageMetadata ignores tagSlice
		tagSlice = nil
	} else {
		switch {
		case HubAPI == false:
			tagSlice, currentMetadataSlice = GetImageMetadata(oldMetadataSet)
		case HubAPI == true:
			tagSlice, currentMetadataSlice = GetImageMetadataTokenAuthV1(oldMetadataSet)
		}
	}

	// get only the new metadata from currentMetadataSlice
	currentMetadataSet = NewMetadataSet()
	for _, metadata := range currentMetadataSlice {
		currentMetadataSet[metadata] = true
		if _, ok := oldMetadataSet[metadata]; !ok {
			// metadata is not in old map
			metadataSlice = append(metadataSlice, metadata)
		}
	}

	// find entries in the old map that are not in the current map,
	// and remove those entries from the database
	obsolete := []ImageMetadataInfo{}
	for metadata := range oldMetadataSet {
		if _, ok := currentMetadataSet[metadata]; !ok {
			if len(ReposToProcess) > 0 {
				if _, present := ReposToProcess[RepoType(metadata.Repo)]; present {
					obsolete = append(obsolete, metadata)
					blog.Info("Need to remove ImageMetadata: %v", metadata)
				}
			} else {
				obsolete = append(obsolete, metadata)
				blog.Info("Need to remove ImageMetadata: %v", metadata)
			}
		}
	}
	if len(obsolete) > 0 {
		RemoveObsoleteMetadata(obsolete)
	}

	if len(metadataSlice) > 0 || len(obsolete) > 0 {
		config.BanyanUpdate("Detected changes in registry metadata")
	}

	// Sort image metadata from newest image to oldest image
	sort.Sort(ByDateTime(metadataSlice))
	return
}

// NewDockerTransport creates an HTTP transport to the Docker unix/tcp socket.
func NewDockerTransport(proto, addr string) (tr *http.Transport, e error) {
	// check Docker environment variables
	dockerHost := os.Getenv("DOCKER_HOST")
	if os.Getenv("DOCKER_TLS_VERIFY") == "0" {
		DockerTLSVerify = false
	}
	dockerCertPath := os.Getenv("DOCKER_CERT_PATH")
	if dockerHost == "" {
		DockerProto = proto
		DockerAddr = addr
	} else {
		blog.Info("$DOCKER_HOST env var = %s", dockerHost)
		switch {
		case strings.HasPrefix(dockerHost, "tcp://"):
			blog.Info("Using protocol tcp")
			DockerProto = "tcp"
			DockerAddr = dockerHost[6:]
		case strings.HasPrefix(dockerHost, "unix://"):
			blog.Info("Using protocol unix")
			DockerProto = "unix"
			DockerAddr = dockerHost[6:]
		default:
			except.Fail("Unexpected value in $DOCKER_HOST:", dockerHost)
		}
	}

	// create transport for unix socket
	if DockerProto != "unix" && DockerProto != "tcp" {
		e = errors.New("Protocol " + DockerProto + " is not yet supported")
		return
	}
	if DockerProto == "unix" {
		tr = &http.Transport{}
		tr.DisableCompression = true
		tr.Dial = func(_, _ string) (net.Conn, error) {
			return net.DialTimeout(DockerProto, DockerAddr, HTTPTIMEOUT)
		}
		return
	}
	if DockerTLSVerify {
		certfile := dockerCertPath + "/cert.pem"
		cafile := dockerCertPath + "/ca.pem"
		keyfile := dockerCertPath + "/key.pem"
		tr, e = NewTLSTransport(DockerAddr, certfile, cafile, keyfile)
		if e != nil {
			except.Fail(e, "NewTLSTransport")
		}
		return
	}

	tr = &http.Transport{}
	return
}

// checkRepoList gets the list of repositories to process from the command line
// and from the repoList file.
func checkRepoList(initial bool) (updates bool) {
	newList := make(map[collector.RepoType]bool)

	// check repositories specified on the command line
	if len(flag.Args()) > 1 {
		for _, repo := range flag.Args()[1:] {
			newList[collector.RepoType(repo)] = true
			if initial {
				updates = true
			}
		}
	}
	// check repositories specified in the repoList file. Ignore file read errors.
	data, err := ioutil.ReadFile(*repoList)
	if err != nil {
		if initial {
			blog.Info("Repolist: " + *repoList + " not specified")
		}
	} else {
		arr := strings.Split(string(data), "\n")
		for _, line := range arr {
			// skip over comments and whitespace
			arr := strings.Split(line, "#")
			repo := arr[0]
			repotrim := strings.TrimSpace(repo)
			if repotrim != "" {
				r := collector.RepoType(repotrim)
				newList[r] = true
				if _, ok := collector.ReposToProcess[r]; !ok {
					updates = true
				}
			}
		}
	}

	if len(newList) == 0 {
		collector.ReposToProcess = newList
		return
	}
	collector.ReposToProcess = newList
	if searchTerm := collector.NeedRegistrySearch(); searchTerm != "" {
		config.FilterRepos = false
	} else {
		config.FilterRepos = true
	}
	if updates {
		blog.Info("Limiting collection to the following repos:")
		for repo := range newList {
			blog.Info(repo)
		}
	}
	return
}

// RemoveImages removes least recently pulled docker images from the local docker host.
func RemoveImages(PulledImages []ImageMetadataInfo) {
	numRemoved := 0
	imageMap, err := GetLocalImages(false, false)
	if err != nil {
		except.Error(err, ": RemoveImages unable to list local images")
	}
	for _, metadata := range PulledImages {
		if strings.HasPrefix(metadata.Repo, "library/") {
			metadata.Repo = strings.Replace(metadata.Repo, "library/", "", 1)
		}
		imageID := ImageIDType(metadata.Image)
		if metadata.Image == "" {
			// unknown image ID. Search the repotags for a match
			var err error
			imageID, err = imageMap.Image(RepoType(metadata.Repo), TagType(metadata.Tag))
			if err != nil {
				except.Error(err, ": RemoveImages unable to find image ID")
				break
			}
		}

		// Get all repo:tags associated with the image
		repoTagSlice := imageMap.RepoTags(imageID)
		if len(repoTagSlice) == 0 {
			except.Error("RemoveImages unable to find expected repo:tag " + metadata.Repo +
				":" + metadata.Tag + " for image ID=" + string(imageID))
			except.Error("imageMap is %v", imageMap)
			continue
		}
		for _, repotag := range repoTagSlice {
			// basespec := RegistrySpec + "/" + string(t.Repo) + ":"
			if ExcludeRepo[RepoType(repotag.Repo)] {
				continue
			}
			apipath := "/images/" + string(repotag.Repo) + ":" + string(repotag.Tag)
			blog.Info("RemoveImages %s", apipath)
			config.BanyanUpdate("Remove", apipath)
			_, err := DockerAPI(DockerTransport, "DELETE", apipath, []byte{}, "")
			if err != nil {
				except.Error(err, "RemoveImages Repo:Tag", repotag.Repo, repotag.Tag,
					"image", metadata.Image)
			}
			numRemoved++
		}
	}

	blog.Info("Number of repo/tags removed this time around: %d", numRemoved)

	RemoveDanglingImages()
	return
}

func main() {
	doFlags()

	setupLogging()

	//verifyVolumes()

	copyBanyanData()

	// setup connection to docker daemon's unix/tcp socket
	var e error
	collector.DockerTransport, e = collector.NewDockerTransport(*dockerProto, *dockerAddr)
	if e != nil {
		except.Fail(e, ": Error in connecting to docker remote API socket")
	}

	authToken := RegisterCollector()

	// Set output writers
	SetOutputWriters(authToken)
	SetupBanyanStatus(authToken)

	checkConfigUpdate(true)
	if collector.LocalHost == false && collector.RegistryAPIURL == "" {
		collector.RegistryAPIURL, collector.HubAPI, collector.BasicAuth, collector.XRegistryAuth = collector.GetRegistryURL()
		blog.Info("registry API URL: %s", collector.RegistryAPIURL)
	}

	// Log the docker version
	major, minor, revision, e := collector.DockerVersion()
	if e != nil {
		except.Error(e, ": Could not identify Docker version")
	} else {
		blog.Info("Docker version %d.%d.%d", major, minor, revision)
		config.BanyanUpdate("Docker version", strconv.Itoa(major)+"."+strconv.Itoa(minor)+"."+strconv.Itoa(revision))
	}

	// Images we have processed already
	processedImages := collector.NewImageSet()
	e = getImageList(processedImages)
	if e != nil {
		blog.Info("Fresh start: No previously collected images were found in %s", *imageList)
	}
	_ = getImageManifestHashList(processedImages)
	blog.Debug(processedImages)

	// Main infinite loop.
	InfLoop(authToken, processedImages)
}

// GetNewImageMetadata takes the set of existing images, queries the registry to find any changes,
// and then brings the Output Writer up to date by telling it the obsolete metadata to delete
// and the new metadata to add.
func GetNewImageMetadata(oldMetadataSet MetadataSet) (metadataSlice []ImageMetadataInfo, currentMetadataSet MetadataSet) {

	var currentMetadataSlice []ImageMetadataInfo
	//config.BanyanUpdate("Loading Registry Metadata")
	if LocalHost == true {
		blog.Info("Collect images from local Docker host")
		currentMetadataSlice = GetLocalImageMetadata(oldMetadataSet)
	} else {
		currentMetadataSlice = GetImageMetadata(oldMetadataSet)
	}

	// get only the new metadata from currentMetadataSlice
	currentMetadataSet = NewMetadataSet()
	for _, metadata := range currentMetadataSlice {
		currentMetadataSet.Insert(metadata)
		if oldMetadataSet.Exists(metadata) == false {
			// metadata is not in old map
			metadataSlice = append(metadataSlice, metadata)
		}
	}

	// find entries in the old map that are not in the current map,
	// and remove those entries from the database
	obsolete := []ImageMetadataInfo{}
	for metadata := range oldMetadataSet {
		if !currentMetadataSet.Exists(metadata) {
			if len(ReposToProcess) > 0 {
				if _, present := ReposToProcess[RepoType(metadata.Repo)]; present {
					obsolete = append(obsolete, metadata)
					blog.Info("Need to remove ImageMetadata: %v", metadata)
				}
			} else {
				obsolete = append(obsolete, metadata)
				blog.Info("Need to remove ImageMetadata: %v", metadata)
			}
		}
	}
	if len(obsolete) > 0 {
		RemoveObsoleteMetadata(obsolete)
	}

	if len(metadataSlice) > 0 || len(obsolete) > 0 {
		config.BanyanUpdate("Detected changes in registry metadata")
	}

	// Sort image metadata from newest image to oldest image
	sort.Sort(ByDateTime(metadataSlice))
	return
}

// WriteImageAllData writes image (pkg and other) data into file
func (f *FileWriter) WriteImageAllData(outMapMap map[string]map[string]interface{}) {
	blog.Info("Writing image (pkg and other) data into file...")

	for imageID, scriptMap := range outMapMap {
		for scriptName, out := range scriptMap {
			scriptDir := f.dir + "/" + trimExtension(scriptName)
			err := fsutil.CreateDirIfNotExist(scriptDir)
			if err != nil {
				blog.Error(err, ": Error creating script dir: ", scriptDir)
				continue
			}
			image := string(imageID)
			if len(image) < 12 {
				blog.Warn("Weird...Haven't seen imageIDs so small -- possibly a test?")
			} else {
				image = string(imageID)[0:12]
			}
			filenamePath := scriptDir + "/" + image
			if _, ok := out.([]byte); ok {
				f.format = "txt"
				filenamePath += "-miscdata"
			} else {
				// by default it is json. But f.format could get overwritten at any point
				// in the for loop if the output type is []byte, hence the (re)assignment
				f.format = "json"
				// NOTE: If we start using json for output other than imageData, change this
				filenamePath += "-pkgdata"
			}
			f.writeFileInFormat(filenamePath, &out)
		}
	}
	return
}

// lookupMetadataTokenAuthV1 takes as input the imageID, and Docker Hub auth/index info,
// and it returns ImageMetadataInfo for that image by querying the indexed registry.
func lookupMetadataTokenAuthV1(imageID ImageIDType, client *http.Client, indexInfo IndexInfo) (
	metadata ImageMetadataInfo, e error) {

	blog.Info("Get Metadata for Image: %s", string(imageID))
	URL := "https://" + indexInfo.RegistryURL + "/v1/images/" + string(imageID) + "/json"
	response, e := RegistryRequestWithToken(client, URL, indexInfo.DockerToken)
	if e != nil {
		blog.Error(e, "Unable to query metadata for image: "+string(imageID))
		return
	}
	// log.Print("metadata query response: " + string(response))
	var m ImageStruct
	if e = json.Unmarshal(response, &m); e != nil {
		return
	}
	var creationTime time.Time
	metadata.Image = string(imageID)
	if creationTime, e = time.Parse(time.RFC3339Nano, m.Created); e != nil {
		return
	}
	metadata.Datetime = creationTime
	metadata.Size = m.Size
	metadata.Author = m.Author
	metadata.Checksum = m.Checksum
	metadata.Comment = m.Comment
	metadata.Parent = m.Parent
	return
}

// GetImageMetadataTokenAuthV1 returns repositories/tags/image metadata from the Docker Hub
// or other registry using v1 token authorization.
// The user must have specified a set of repositories of interest.
// The function queries the index server, e.g., Docker Hub, to get the token and registry, and then uses
// the token to query the registry.
func GetImageMetadataTokenAuthV1(oldMetadataSet MetadataSet) (tagSlice []TagInfo, metadataSlice []ImageMetadataInfo) {
	if len(ReposToProcess) == 0 {
		return
	}
	client := &http.Client{}

	metadataMap := NewImageToMetadataMap(oldMetadataSet)

	for repo := range ReposToProcess {
		blog.Info("Get index and tag info for %s", string(repo))
		config.BanyanUpdate("Get index and tag info for", string(repo))

		var (
			indexInfo         IndexInfo
			e                 error
			repoTagSlice      []TagInfo
			repoMetadataSlice []ImageMetadataInfo
		)

		// loop until success
		for {
			indexInfo, e = getReposTokenAuthV1(repo, client)
			if e != nil {
				blog.Warn(e, ":index lookup failed for repo", string(repo), "- retrying.")
				config.BanyanUpdate(e.Error(), ":index lookup failed, repo", string(repo), "- retrying")
				time.Sleep(config.RETRYDURATION)
				continue
			}

			repoTagSlice, e = getTagsTokenAuthV1(repo, client, indexInfo)
			if e != nil {
				blog.Warn(e, ":tag lookup failed for repo", string(repo), "- retrying.")
				config.BanyanUpdate(e.Error(), ":tag lookup failed for repo", string(repo), "- retrying")
				time.Sleep(config.RETRYDURATION)
				continue
			}
			if len(repoTagSlice) != 1 {
				blog.Error("Incorrect length of repoTagSlice: expected length=1, got length=%d", len(repoTagSlice))
				config.BanyanUpdate("Incorrect length of repoTagSlice:", strconv.Itoa(len(repoTagSlice)), string(repo))
				time.Sleep(config.RETRYDURATION)
				continue
			}

			repoMetadataSlice, e = getMetadataTokenAuthV1(repoTagSlice[0], metadataMap, client, indexInfo)
			if e != nil {
				blog.Warn(e, ":metadata lookup failed for", string(repoTagSlice[0].Repo), "- retrying.")
				config.BanyanUpdate(e.Error(), ":metadata lookup failed for", string(repoTagSlice[0].Repo), "- retrying")
				time.Sleep(config.RETRYDURATION)
				continue
			}
			//success!
			break
		}
		tagSlice = append(tagSlice, repoTagSlice...)
		metadataSlice = append(metadataSlice, repoMetadataSlice...)
	}
	return
}

func main() {
	doFlags()

	setupLogging()

	//verifyVolumes()

	copyBanyanData()

	// setup connection to docker daemon's unix/tcp socket
	var e error
	collector.DockerTransport, e = collector.NewDockerTransport(*dockerProto, *dockerAddr)
	if e != nil {
		blog.Exit(e, ": Error in connecting to docker remote API socket")
	}

	authToken := RegisterCollector()

	// Set output writers
	SetOutputWriters(authToken)
	SetupBanyanStatus(authToken)

	checkConfigUpdate(true)
	if collector.LocalHost == false && collector.RegistryAPIURL == "" {
		collector.RegistryAPIURL, collector.HubAPI, collector.BasicAuth, collector.XRegistryAuth = collector.GetRegistryURL()
		blog.Info("registry API URL: %s", collector.RegistryAPIURL)
	}

	// Images we have processed already
	processedImages := collector.NewImageSet()
	e = getImageList(processedImages)
	if e != nil {
		blog.Info("Fresh start: No previously collected images were found in %s", *imageList)
	}
	blog.Debug(processedImages)

	// Image Metadata we have already seen
	MetadataSet := collector.NewMetadataSet()
	PulledList := []collector.ImageMetadataInfo{}

	// Main infinite loop.
	InfLoop(authToken, processedImages, MetadataSet, PulledList)
}

// DockerAPI performs an HTTP GET,POST,DELETE operation to the Docker daemon.
func DockerAPI(tr *http.Transport, operation, apipath string, jsonString []byte,
	XRegistryAuth string) (resp []byte, e error) {
	switch operation {
	case "GET", "POST", "DELETE":
		break
	default:
		e = errors.New("Operation " + operation + " not supported")
		return
	}
	// for unix socket, URL (host.domain) is needed but can be anything
	var host string
	HTTP := "http://"
	if DockerProto == "unix" {
		host = dummydomain
	} else {
		host = DockerAddr
		if DockerTLSVerify {
			HTTP = "https://"
		}
	}
	URL := HTTP + host + apipath
	blog.Info("DockerAPI %s", URL)
	req, e := http.NewRequest(operation, URL, bytes.NewBuffer(jsonString))
	if e != nil {
		except.Error(e, ":DockerAPI failed to create http request")
		return
	}
	req.Header.Add("Content-Type", "application/json")
	if XRegistryAuth != "" {
		req.Header.Add("X-Registry-Auth", XRegistryAuth)
	}

	//req.Header.Set("Authorization", "Bearer "+authToken)
	client := &http.Client{Transport: tr, Timeout: DockerTimeout}
	r, e := client.Do(req)
	if e != nil {
		except.Error(e, ":DockerAPI URL", URL, "client request failed")
		return
	}
	defer r.Body.Close()
	resp, e = ioutil.ReadAll(r.Body)
	if e != nil {
		except.Error(e, ":DockerAPI URL", URL, "invalid response body")
		return
	}
	if r.StatusCode < 200 || r.StatusCode > 299 {
		e = errors.New("DockerAPI URL: " + URL + " status code: " + strconv.Itoa(r.StatusCode) +
			"error: " + string(resp))
		return
	}
	return
}

// createCmd returns a json byte slice desribing the container we want to create
func createCmd(imageID ImageIDType, scriptName, staticBinary, dirPath string) (jsonString []byte, err error) {
	var container Container
	container.User = "0"
	container.AttachStdout = true
	container.AttachStderr = true
	container.HostConfig.Binds = []string{config.BANYANHOSTDIR() + "/hosttarget" + ":" + TARGETCONTAINERDIR + ":ro"}
	container.Image = string(imageID)

	container.Entrypoint = []string{TARGETCONTAINERDIR + "/bin/bash-static", "-c"}
	container.Cmd = []string{"PATH=" + TARGETCONTAINERDIR + "/bin" + ":$PATH " + staticBinary + " " + dirPath + "/" + scriptName}
	blog.Info("Executing command: docker %v", container.Cmd)
	return json.Marshal(container)
}

// TestWriteImageMetadata tests writing (appending/removing) imageMD to file
func TestWriteImageMetadata(t *testing.T) {
	const (
		format  = "json"
		destDir = "/tmp"
	)

	// Testing imagedata...
	var imdata = []ImageMetadataInfo{
		{"111", time.Now(), "r1", "t1", 100, "a1", "c1", "c1", "p1"},
		{"121", time.Now(), "r2", "t2", 100, "a2", "c2", "c2", "p2"},
		{"131", time.Now(), "r3", "t3", 100, "a3", "c3", "c3", "p3"},
	}

	// Remove output file if it already exists -- since we append
	file := "metadata." + format
	filenamePath := destDir + "/" + file
	if _, err := os.Stat(filenamePath); err == nil {
		// file exists
		e := os.Remove(filenamePath)
		if e != nil {
			t.Fatal(": Error in removing metadata file: ", filenamePath)
		}
	} //ignore else

	// Append to MD file
	b1 := testWriteImageMDToFile(t, imdata, "/tmp", "json", "ADD")
	data1 := ImageMetadataAndAction{"ADD", imdata}
	b2, err := json.MarshalIndent(data1, "", "\t")
	if err != nil {
		t.Fatal(err, ": Error in marshaling json")
	}
	if !bytes.Equal(b1, b2) {
		t.Fatal("Input/Output image metadata don't match")
	}

	// "Remove" from MD file (note that action is set to remove, rather than really removing anything)
	b3 := testWriteImageMDToFile(t, []ImageMetadataInfo{imdata[0]}, "/tmp", "json", "REMOVE")
	data2 := ImageMetadataAndAction{"REMOVE", []ImageMetadataInfo{imdata[0]}}
	b4, err := json.MarshalIndent(data2, "", "\t")
	b5 := append(b2, b4...)
	if err != nil {
		t.Fatal(err, ": Error in marshaling json")
	}
	if !bytes.Equal(b3, b5) {
		blog.Info(string(b5))
		t.Fatal("Input/Output image metadata don't match: ", len(b3), len(b5))
	}
	//Pass...
	return
}

// GetLocalImageMetadata returns image metadata queried from a local Docker host.
// Query the local docker daemon to detect new image builds on the host and new images pulled from registry by users.
func GetLocalImageMetadata(oldMetadataSet MetadataSet) (metadataSlice []ImageMetadataInfo) {
	for {
		blog.Info("Get a list of images from local Docker daemon")
		imageMap, e := GetLocalImages()
		if e != nil {
			blog.Warn(e, " GetLocalImages")
			blog.Warn("Retrying")
			time.Sleep(config.RETRYDURATION)
			continue
		}

		blog.Info("Get Image Metadata from local Docker daemon")
		// Get image metadata
		metadataSlice, e = getImageMetadata(imageMap, oldMetadataSet)
		if e != nil {
			blog.Warn(e, " GetImageMetadata")
			blog.Warn("Retrying")
			time.Sleep(config.RETRYDURATION)
			continue
		}
		break
	}
	return
}