Golang util.Retry函数代码示例

// Test fleet running through an etcd2 proxy.
func Proxy(c platform.TestCluster) error {
	masterconf.CoreOS.Etcd2.Discovery, _ = c.GetDiscoveryURL(1)
	master, err := c.NewMachine(masterconf.String())
	if err != nil {
		return fmt.Errorf("Cluster.NewMachine: %s", err)
	}
	defer master.Destroy()

	proxyconf.CoreOS.Etcd2.Discovery = masterconf.CoreOS.Etcd2.Discovery
	proxy, err := c.NewMachine(proxyconf.String())
	if err != nil {
		return fmt.Errorf("Cluster.NewMachine: %s", err)
	}
	defer proxy.Destroy()

	err = platform.InstallFile(strings.NewReader(fleetunit), proxy, "/home/core/hello.service")
	if err != nil {
		return fmt.Errorf("InstallFile: %s", err)
	}

	// settling...
	fleetStart := func() error {
		_, err = proxy.SSH("fleetctl start /home/core/hello.service")
		if err != nil {
			return fmt.Errorf("fleetctl start: %s", err)
		}
		return nil
	}
	if err := util.Retry(5, 5*time.Second, fleetStart); err != nil {
		return fmt.Errorf("fleetctl start failed: %v", err)
	}

	var status []byte

	fleetList := func() error {
		status, err = proxy.SSH("fleetctl list-units -l -fields active -no-legend")
		if err != nil {
			return fmt.Errorf("fleetctl list-units: %s", err)
		}

		if !bytes.Equal(status, []byte("active")) {
			return fmt.Errorf("unit not active")
		}

		return nil
	}

	if err := util.Retry(5, 1*time.Second, fleetList); err != nil {
		return fmt.Errorf("fleetctl list-units failed: %v", err)
	}

	return nil
}

func checkEtcdVersion(cluster platform.Cluster, m platform.Machine, expected string) error {
	var b []byte
	var err error

	checker := func() error {
		cmd := cluster.NewCommand("curl", "-L", fmt.Sprintf("http://%v:2379/version", m.IP()))
		b, err = cmd.Output()
		if err != nil {
			return fmt.Errorf("curl failed: %v", err)
		}

		return nil
	}

	if err := util.Retry(15, 10*time.Second, checker); err != nil {
		return err
	}

	plog.Infof("got version: %s", b)

	if string(b) != expected {
		return fmt.Errorf("expected %v, got %s", expected, b)
	}

	return nil
}

func checkEtcdVersion(cluster platform.Cluster, m platform.Machine, expected string) error {
	var b []byte

	checker := func() error {
		out, err := m.SSH(fmt.Sprintf("curl -s -L http://%s:2379/version", m.IP()))
		if err != nil {
			return fmt.Errorf("curl failed: %v", out)
		}

		b = out

		return nil
	}

	if err := util.Retry(15, 10*time.Second, checker); err != nil {
		return err
	}

	plog.Infof("got version: %s", b)

	if string(b) != expected {
		return fmt.Errorf("expected %v, got %s", expected, b)
	}

	return nil
}

// poll cluster-health until result
func getClusterHealth(m platform.Machine, csize int) error {
	var err error
	var b []byte

	checker := func() error {
		b, err := m.SSH("etcdctl cluster-health")
		if err != nil {
			return err
		}

		// repsonse should include "healthy" for each machine and for cluster
		if strings.Count(string(b), "healthy") != (csize*2)+1 {
			return fmt.Errorf("unexpected etcdctl output")
		}

		plog.Infof("cluster healthy")
		return nil
	}

	err = util.Retry(15, 10*time.Second, checker)
	if err != nil {
		return fmt.Errorf("health polling failed: %v: %s", err, b)
	}

	return nil
}

func sshCheck(gm *gceMachine) error {
	var err error

	// Allow a few authentication failures in case setup is slow.
	sshchecker := func() error {
		gm.sshClient, err = gm.gc.sshAgent.NewClient(gm.IP())
		if err != nil {
			return err
		}
		return nil
	}

	if err := util.Retry(sshRetries, sshTimeout, sshchecker); err != nil {
		return err
	}

	// sanity check
	out, err := gm.SSH("grep ^ID= /etc/os-release")
	if err != nil {
		return err
	}

	if !bytes.Equal(out, []byte("ID=coreos")) {
		return fmt.Errorf("Unexpected SSH output: %s", out)
	}

	return nil
}

func nginxCheck(master platform.Machine, nodes []platform.Machine) error {
	pod := strings.NewReader(nginxPodYAML)
	if err := platform.InstallFile(pod, master, "./nginx-pod.yaml"); err != nil {
		return err
	}
	if _, err := master.SSH("./kubectl create -f nginx-pod.yaml"); err != nil {
		return err
	}
	// wait for pod status to be 'Running'
	podIsRunning := func() error {
		b, err := master.SSH("./kubectl get pod nginx -o=template -t={{.status.phase}}")
		if err != nil {
			return err
		}
		if !bytes.Equal(b, []byte("Running")) {
			return fmt.Errorf("nginx pod not running: %s", b)
		}
		return nil
	}
	if err := util.Retry(10, 5*time.Second, podIsRunning); err != nil {
		return err
	}

	// delete pod
	_, err := master.SSH("./kubectl delete pods nginx")
	if err != nil {
		return err
	}

	return nil
}

func discovery(cluster platform.Cluster, version int) error {
	if plog.LevelAt(capnslog.DEBUG) {
		// get journalctl -f from all machines before starting
		for _, m := range cluster.Machines() {
			if err := platform.StreamJournal(m); err != nil {
				return fmt.Errorf("failed to start journal: %v", err)
			}
		}
	}

	// start etcd on each machine asynchronously.
	for _, m := range cluster.Machines() {
		if err := doStart(m, version, false); err != nil {
			return err
		}
	}

	// block until each instance is reported as started.
	for i, m := range cluster.Machines() {
		if err := doStart(m, version, true); err != nil {
			return err
		}
		plog.Infof("etcd instance%d started", i)
	}

	var keyMap map[string]string
	var retryFuncs []func() error

	retryFuncs = append(retryFuncs, func() error {
		var err error
		keyMap, err = SetKeys(cluster, 5)
		if err != nil {
			return err
		}
		return nil
	})
	retryFuncs = append(retryFuncs, func() error {
		var quorumRead bool
		if version == 2 {
			quorumRead = true
		}
		if err := CheckKeys(cluster, keyMap, quorumRead); err != nil {
			return err
		}
		return nil
	})
	for _, retry := range retryFuncs {
		if err := util.Retry(5, 5*time.Second, retry); err != nil {
			return fmt.Errorf("discovery failed health check: %v", err)
		}
		// NOTE(pb): etcd1 seems to fail in an odd way when I try quorum
		// read, instead just sleep between setting and getting.
		time.Sleep(2 * time.Second)
	}

	return nil
}

// Start a multi-node cluster from offcial kubernetes 1.0 guides. Once
// up, do a couple basic smoke checks. See:
// http://kubernetes.io/v1.0/docs/getting-started-guides/coreos/coreos_multinode_cluster.html
func MultiNodeSmoke(c platform.TestCluster) error {
	const clusterSize = 3

	// spawn master
	master, err := c.NewMachine(masterConfig)
	if err != nil {
		return err
	}

	// get master private IP and place into nodeConfig
	nodeConfig = strings.Replace(nodeConfig, "<master-private-ip>", master.PrivateIP(), -1)
	var nodeConfigs []string
	for i := 0; i < clusterSize-1; i++ {
		nodeConfigs = append(nodeConfigs, nodeConfig)
	}

	// spawn nodes
	nodes, err := platform.NewMachines(c, nodeConfigs)
	if err != nil {
		return err
	}

	// get kubectl in master
	_, err = master.SSH("wget -q https://storage.googleapis.com/kubernetes-release/release/v1.0.1/bin/linux/amd64/kubectl")
	if err != nil {
		return err
	}
	_, err = master.SSH("chmod +x kubectl")
	if err != nil {
		return err
	}

	// check that all nodes appear in kubectl
	f := func() error {
		if err = nodeCheck(master, nodes); err != nil {
			return err
		}
		return nil
	}
	if err := util.Retry(10, 5*time.Second, f); err != nil {
		return err
	}

	// start nginx pod and curl endpoint
	if err = nginxCheck(master, nodes); err != nil {
		return err
	}

	// http://kubernetes.io/v1.0/docs/user-guide/secrets/ Also, ensures
	// https://github.com/coreos/bugs/issues/447 does not re-occur.
	if err = secretCheck(master, nodes); err != nil {
		return err
	}

	return nil
}

// JournalRemote tests that systemd-journal-remote can read log entries from
// a systemd-journal-gatewayd server.
func JournalRemote(c platform.TestCluster) error {
	// start gatewayd and log a message
	gateway, err := c.NewMachine(gatewayconf.String())
	if err != nil {
		return fmt.Errorf("Cluster.NewMachine: %s", err)
	}
	defer gateway.Destroy()

	// log a unique message on gatewayd machine
	msg := "supercalifragilisticexpialidocious"
	out, err := gateway.SSH("logger " + msg)
	if err != nil {
		return fmt.Errorf("logger: %v: %v", out, err)
	}

	// spawn a machine to read from gatewayd
	collector, err := c.NewMachine("")
	if err != nil {
		return fmt.Errorf("Cluster.NewMachine: %s", err)
	}
	defer collector.Destroy()

	// collect logs from gatewayd machine
	cmd := fmt.Sprintf("sudo systemd-run --unit systemd-journal-remote-client /usr/lib/systemd/systemd-journal-remote --url http://%s:19531", gateway.PrivateIP())
	out, err = collector.SSH(cmd)
	if err != nil {
		return fmt.Errorf("failed to start systemd-journal-remote: %v: %v", out, err)
	}

	// find the message on the collector
	journalReader := func() error {
		cmd = fmt.Sprintf("sudo journalctl _HOSTNAME=%s -t core --file /var/log/journal/remote/remote-%s:19531.journal", gatewayconf.Hostname, gateway.PrivateIP())
		out, err = collector.SSH(cmd)
		if err != nil {
			return fmt.Errorf("journalctl: %v: %v", out, err)
		}

		if !strings.Contains(string(out), msg) {
			return fmt.Errorf("journal missing entry: expected %q got %q", msg, out)
		}

		return nil
	}

	if err := util.Retry(5, 2*time.Second, journalReader); err != nil {
		return err
	}

	return nil
}

func DownloadFile(file, url string) error {
	plog.Infof("Downloading %s to %s", url, file)

	if err := os.MkdirAll(filepath.Dir(file), 0777); err != nil {
		return err
	}

	download := func() error {
		return downloadFile(file, url)
	}
	if err := util.Retry(5, 1*time.Second, download); err != nil {
		return err
	}
	return nil
}

// run clustering based tests
func ClusterTests(c platform.TestCluster) error {
	if plog.LevelAt(capnslog.DEBUG) {
		// get journalctl -f from all machines before starting
		for _, m := range c.Machines() {
			if err := platform.StreamJournal(m); err != nil {
				return fmt.Errorf("failed to start journal: %v", err)
			}
		}
	}

	// make sure etcd is up and running
	var keyMap map[string]string
	var retryFuncs []func() error

	retryFuncs = append(retryFuncs, func() error {
		var err error
		keyMap, err = etcd.SetKeys(c, 3)
		if err != nil {
			return err
		}
		return nil
	})
	retryFuncs = append(retryFuncs, func() error {
		if err := etcd.CheckKeys(c, keyMap, true); err != nil {
			return err
		}
		return nil
	})
	for _, retry := range retryFuncs {
		if err := util.Retry(5, 5*time.Second, retry); err != nil {
			return fmt.Errorf("etcd failed health check: %v", err)
		}
	}

	tests := c.ListNativeFunctions()
	for _, name := range tests {
		plog.Noticef("running %v...", name)
		err := c.RunNative(name, c.Machines()[0])
		if err != nil {
			return err
		}
	}
	return nil

}

// Test that timesyncd starts using the local NTP server
func NTP(c platform.TestCluster) error {
	m, err := c.NewMachine("")
	if err != nil {
		return fmt.Errorf("Cluster.NewMachine: %s", err)
	}
	defer m.Destroy()

	out, err := m.SSH("networkctl status eth0")
	if err != nil {
		return fmt.Errorf("networkctl: %v", err)
	}
	if !bytes.Contains(out, []byte("NTP: 10.0.0.1")) {
		return fmt.Errorf("Bad network config:\n%s", out)
	}

	plog.Info("Waiting for systemd-timesyncd.service")

	checker := func() error {
		out, err = m.SSH("systemctl status systemd-timesyncd.service")
		if err != nil {
			return fmt.Errorf("systemctl: %v", err)
		}

		if !bytes.Contains(out, []byte(`Status: "Synchronized to time server 10.0.0.1:123 (10.0.0.1)."`)) {
			return fmt.Errorf("unexpected systemd-timesyncd status: %v", out)
		}

		plog.Info("systemd-timesyncd.service is working!")
		return nil
	}

	err = util.Retry(60, 1*time.Second, checker)
	if err != nil {
		return nil
	}

	return nil
}

func DownloadFile(file, fileURL string, client *http.Client) error {
	plog.Infof("Downloading %s to %s", fileURL, file)

	// handle bucket urls by using api to get media link
	parseURL, err := url.Parse(fileURL)
	if err != nil {
		return err
	}
	if parseURL.Scheme == "gs" {
		if client == nil {
			client = http.DefaultClient
		}
		api, err := storage.New(client)
		if err != nil {
			plog.Fatal(err)
		}
		path := strings.TrimLeft(parseURL.Path, "/")
		obj, err := api.Objects.Get(parseURL.Host, path).Do()
		if err != nil {
			plog.Fatal(err)
		}
		fileURL = obj.MediaLink
	}

	if err := os.MkdirAll(filepath.Dir(file), 0777); err != nil {
		return err
	}

	download := func() error {
		return downloadFile(file, fileURL, client)
	}
	if err := util.Retry(5, 1*time.Second, download); err != nil {
		return err
	}
	return nil
}

// commonMachineChecks tests a machine for various error conditions such as ssh
// being available and no systemd units failing at the time ssh is reachable.
// It also ensures the remote system is running CoreOS.
//
// TODO(mischief): better error messages.
func commonMachineChecks(m Machine) error {
	// ensure ssh works
	sshChecker := func() error {
		_, err := m.SSH("true")
		if err != nil {
			return err
		}
		return nil
	}

	if err := util.Retry(sshRetries, sshTimeout, sshChecker); err != nil {
		return fmt.Errorf("ssh unreachable: %v", err)
	}

	// ensure we're talking to a CoreOS system
	out, err := m.SSH("grep ^ID= /etc/os-release")
	if err != nil {
		return fmt.Errorf("no /etc/os-release file")
	}

	if !bytes.Equal(out, []byte("ID=coreos")) {
		return fmt.Errorf("not a CoreOS instance")
	}

	// ensure no systemd units failed during boot
	out, err = m.SSH("systemctl --no-legend --state failed list-units")
	if err != nil {
		return fmt.Errorf("systemctl: %v: %v", out, err)
	}

	if len(out) > 0 {
		return fmt.Errorf("some systemd units failed:\n%s", out)
	}

	return nil
}

func runUpdatePayload(cmd *cobra.Command, args []string) {
	if len(args) != 0 {
		plog.Fatal("No args accepted")
	}

	plog.Info("Generating update payload")

	// check for update file, generate if it doesn't exist
	version := "latest"
	dir := sdk.BuildImageDir(version)
	payload := "coreos_production_update.gz"

	_, err := os.Stat(filepath.Join(dir, payload))
	if err != nil {
		err = sdkomaha.GenerateFullUpdate("latest", true)
		if err != nil {
			plog.Fatalf("Building full update failed: %v", err)
		}
	}

	plog.Info("Bringing up test harness cluster")

	cluster, err := platform.NewQemuCluster(kola.QEMUOptions)
	qc := cluster.(*platform.QEMUCluster)
	if err != nil {
		plog.Fatalf("Cluster failed: %v", err)
	}
	defer cluster.Destroy()

	svc := &updateServer{
		updatePath: dir,
		payload:    payload,
	}

	qc.OmahaServer.Updater = svc

	// tell omaha server to handle file requests for the images dir
	qc.OmahaServer.Mux.Handle(dir+"/", http.StripPrefix(dir+"/", http.FileServer(http.Dir(dir))))

	_, port, err := net.SplitHostPort(qc.OmahaServer.Addr().String())
	if err != nil {
		plog.Errorf("SplitHostPort failed: %v", err)
		return
	}

	tmplVals := map[string]string{
		"Server": fmt.Sprintf("10.0.0.1:%s", port),
	}

	tmpl := template.Must(template.New("userdata").Parse(userdata))
	buf := new(bytes.Buffer)

	err = tmpl.Execute(buf, tmplVals)
	if err != nil {
		plog.Errorf("Template execution failed: %v", err)
		return
	}

	plog.Infof("Spawning test machine")

	m, err := cluster.NewMachine(buf.String())
	if err != nil {
		plog.Errorf("Machine failed: %v", err)
		return
	}

	plog.Info("Checking for boot from USR-A partition")

	/* check that we are on USR-A. */
	if err := checkUsrPartition(m, []string{"PARTUUID=" + sdk.USRAUUID.String(), "PARTLABEL=USR-A"}); err != nil {
		plog.Errorf("Did not find USR-A partition: %v", err)
		return
	}

	plog.Infof("Triggering update_engine")

	/* trigger update, monitor the progress. */
	out, err := m.SSH("update_engine_client -check_for_update")
	if err != nil {
		plog.Errorf("Executing update_engine_client failed: %v: %v", out, err)
		return
	}

	checker := func() error {
		envs, err := m.SSH("update_engine_client -status 2>/dev/null")
		if err != nil {
			return err
		}

		em := splitNewlineEnv(string(envs))

		if em["CURRENT_OP"] != "UPDATE_STATUS_UPDATED_NEED_REBOOT" {
			return fmt.Errorf("have not arrived in reboot state: currently at %s", em["CURRENT_OP"])
		}

		return nil
	}

	if err := util.Retry(12, 10*time.Second, checker); err != nil {
		plog.Errorf("Applying update payload failed: %v", err)
//.........这里部分代码省略.........