Golang ZkZone.ForSortedClusters方法代码示例

func (this *Clusters) printSummary(zkzone *zk.ZkZone, clusterPattern string, port string) {
	lines := []string{"Zone|Cluster|Brokers|Topics|Partitions|FlatMsg|Cum"}

	type summary struct {
		zone, cluster               string
		brokers, topics, partitions int
		flat, cum                   int64
	}
	summaries := make([]summary, 0, 10)
	zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
		if !patternMatched(zkcluster.Name(), clusterPattern) {
			return
		}

		brokers, topics, partitions, flat, cum := this.clusterSummary(zkcluster)
		summaries = append(summaries, summary{zkzone.Name(), zkcluster.Name(), brokers, topics, partitions,
			flat, cum})

	})
	sortutil.DescByField(summaries, "cum")
	var totalFlat, totalCum int64
	for _, s := range summaries {
		lines = append(lines, fmt.Sprintf("%s|%s|%d|%d|%d|%s|%s",
			s.zone, s.cluster, s.brokers, s.topics, s.partitions,
			gofmt.Comma(s.flat), gofmt.Comma(s.cum)))

		totalCum += s.cum
		totalFlat += s.flat
	}
	this.Ui.Output(columnize.SimpleFormat(lines))
	this.Ui.Output(fmt.Sprintf("Flat:%s Cum:%s", gofmt.Comma(totalFlat), gofmt.Comma(totalCum)))
}

func (this *Clusters) printRegisteredBrokers(zkzone *zk.ZkZone) {
	this.Ui.Output(zkzone.Name())
	zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
		info := zkcluster.RegisteredInfo()
		this.Ui.Output(fmt.Sprintf("    %s(%s)", info.Name(), info.Nickname))
		registeredBrokers := info.Roster
		if len(registeredBrokers) == 0 {
			this.Ui.Warn("        brokers not registered")
		} else {
			for _, b := range registeredBrokers {
				if this.ipInNumber {
					this.Ui.Output(fmt.Sprintf("        %2d %s", b.Id, b.Addr()))
				} else {
					this.Ui.Output(fmt.Sprintf("        %2d %s", b.Id, b.NamedAddr()))
				}
			}
		}

	})
}

func (this *Consumers) printConsumersByHost(zkzone *zk.ZkZone, clusterPattern string) {
	outputs := make(map[string]map[string]map[string]int) // host: {cluster: {topic: count}}

	this.Ui.Output(color.Blue(zkzone.Name()))

	zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
		if !patternMatched(zkcluster.Name(), clusterPattern) {
			return
		}

		consumerGroups := zkcluster.ConsumerGroups()
		for _, group := range consumerGroups {
			for _, c := range group {
				if _, present := outputs[c.Host()]; !present {
					outputs[c.Host()] = make(map[string]map[string]int)
				}

				if _, present := outputs[c.Host()][zkcluster.Name()]; !present {
					outputs[c.Host()][zkcluster.Name()] = make(map[string]int)
				}

				for topic, count := range c.Subscription {
					outputs[c.Host()][zkcluster.Name()][topic] += count
				}
			}
		}

	})

	sortedHosts := make([]string, 0, len(outputs))
	for host, _ := range outputs {
		sortedHosts = append(sortedHosts, host)
	}
	sort.Strings(sortedHosts)
	for _, host := range sortedHosts {
		tc := outputs[host]
		this.Ui.Output(fmt.Sprintf("%s %+v", color.Green("%22s", host), tc))
	}
}

func (this *Topics) printSummary(zkzone *zk.ZkZone, clusterPattern string) {
	lines := []string{"Zone|Cluster|Topic|Partitions|FlatMsg|Cum"}

	var totalFlat, totalCum int64
	zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
		if !patternMatched(zkcluster.Name(), clusterPattern) {
			return
		}

		summaries := this.clusterSummary(zkcluster)
		sortutil.DescByField(summaries, "cum")
		for _, s := range summaries {
			lines = append(lines, fmt.Sprintf("%s|%s|%s|%d|%s|%s",
				s.zone, s.cluster, s.topic, s.partitions, gofmt.Comma(s.flat), gofmt.Comma(s.cum)))

			totalCum += s.cum
			totalFlat += s.flat
		}

	})

	this.Ui.Output(columnize.SimpleFormat(lines))
	this.Ui.Output(fmt.Sprintf("Flat:%s Cum:%s", gofmt.Comma(totalFlat), gofmt.Comma(totalCum)))
}

func (this *Clusters) printClusters(zkzone *zk.ZkZone, clusterPattern string, port string) {
	if this.registeredBrokers {
		this.printRegisteredBrokers(zkzone)
		return
	}

	type clusterInfo struct {
		name, path         string
		nickname           string
		topicN, partitionN int
		err                string
		priority           int
		public             bool
		retention          int
		replicas           int
		brokerInfos        []zk.BrokerInfo
	}
	clusters := make([]clusterInfo, 0)
	zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
		if !patternMatched(zkcluster.Name(), clusterPattern) {
			return
		}

		ci := clusterInfo{
			name: zkcluster.Name(),
			path: zkcluster.Chroot(),
		}
		if this.neat {
			clusters = append(clusters, ci)
			return
		}

		// verbose mode, will calculate topics and partition count
		brokerList := zkcluster.BrokerList()
		if len(brokerList) == 0 {
			ci.err = "no live brokers"
			clusters = append(clusters, ci)
			return
		}

		if port != "" {
			for _, hostport := range brokerList {
				_, p, err := net.SplitHostPort(hostport)
				swallow(err)

				if p != port {
					return
				}
			}
		}

		info := zkcluster.RegisteredInfo()
		if this.publicOnly && !info.Public {
			return
		}

		if !this.verbose {
			ci.brokerInfos = info.Roster
			clusters = append(clusters, ci)
			return
		}

		kfk, err := sarama.NewClient(brokerList, saramaConfig())
		if err != nil {
			ci.err = err.Error()
			clusters = append(clusters, ci)
			return
		}
		topics, err := kfk.Topics()
		if err != nil {
			ci.err = err.Error()
			clusters = append(clusters, ci)
			return
		}
		partitionN := 0
		for _, topic := range topics {
			partitions, err := kfk.Partitions(topic)
			if err != nil {
				ci.err = err.Error()
				clusters = append(clusters, ci)
				continue
			}

			partitionN += len(partitions)
		}

		clusters = append(clusters, clusterInfo{
			name:        zkcluster.Name(),
			nickname:    info.Nickname,
			path:        zkcluster.Chroot(),
			topicN:      len(topics),
			partitionN:  partitionN,
			retention:   info.Retention,
			public:      info.Public,
			replicas:    info.Replicas,
			priority:    info.Priority,
			brokerInfos: info.Roster,
		})
	})

//.........这里部分代码省略.........

func (this *Topology) displayZoneTopology(zkzone *zk.ZkZone) {
	this.Ui.Output(zkzone.Name())

	// {cluster: {topic: brokerHostInfo}}
	brokerInstances := make(map[string]map[string]*brokerHostInfo)

	zkzone.ForSortedBrokers(func(cluster string, liveBrokers map[string]*zk.BrokerZnode) {
		if len(liveBrokers) == 0 {
			this.Ui.Warn(fmt.Sprintf("empty brokers in cluster[%s]", cluster))
			return
		}
		if this.cluster != "" && this.cluster != cluster {
			return
		}

		brokerInstances[cluster] = make(map[string]*brokerHostInfo)

		for _, broker := range liveBrokers {
			if !patternMatched(broker.Host, this.hostPattern) {
				continue
			}

			if _, present := brokerInstances[cluster][broker.Host]; !present {
				brokerInstances[cluster][broker.Host] = newBrokerHostInfo()
			}
			brokerInstances[cluster][broker.Host].addPort(broker.Port, broker.Uptime())
		}

		// find how many partitions a broker is leading
		zkcluster := zkzone.NewCluster(cluster)
		brokerList := zkcluster.BrokerList()
		if len(brokerList) == 0 {
			this.Ui.Warn(fmt.Sprintf("empty brokers in cluster[%s]", cluster))
			return
		}

		kfk, err := sarama.NewClient(brokerList, sarama.NewConfig())
		if err != nil {
			this.Ui.Error(color.Red("    %+v %s", brokerList, err.Error()))
			return
		}

		topics, err := kfk.Topics()
		swallow(err)
		for _, topic := range topics {
			partions, err := kfk.WritablePartitions(topic)
			swallow(err)
			for _, partitionID := range partions {
				leader, err := kfk.Leader(topic, partitionID)
				swallow(err)
				host, _, err := net.SplitHostPort(leader.Addr())
				swallow(err)
				if !patternMatched(host, this.hostPattern) {
					continue
				}

				latestOffset, err := kfk.GetOffset(topic, partitionID, sarama.OffsetNewest)
				if err != nil {
					this.Ui.Error(fmt.Sprintf("%s %s %v", cluster, topic, err))
					continue
				}
				oldestOffset, err := kfk.GetOffset(topic, partitionID, sarama.OffsetOldest)
				if err != nil {
					this.Ui.Error(fmt.Sprintf("%s %s %v", cluster, topic, err))
					continue
				}

				brokerInstances[cluster][host].topicMsgs[topic] += (latestOffset - oldestOffset)
				brokerInstances[cluster][host].addTopicPartition(topic, partitionID)
			}
		}
	})

	hosts := make(map[string]struct{})
	zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
		for host, _ := range brokerInstances[zkcluster.Name()] {
			hosts[host] = struct{}{}
		}
	})
	sortedHosts := make([]string, 0)
	for host, _ := range hosts {
		sortedHosts = append(sortedHosts, host)
	}
	sort.Strings(sortedHosts)

	// sort by host ip
	sortedClusters := make([]string, 0, len(brokerInstances))
	for c, _ := range brokerInstances {
		sortedClusters = append(sortedClusters, c)
	}
	sort.Strings(sortedClusters)

	portN := 0
	hostN := 0
	topicN := 0
	partitionN := 0
	for _, host := range sortedHosts {
		tn := 0
		pn := 0
		mn := int64(0)
//.........这里部分代码省略.........

func (this *Consumers) cleanupStaleConsumerGroups(zkzone *zk.ZkZone, clusterPattern string) {
	// what consumer groups are safe to delete?
	// 1. not online
	// 2. have no offsets
	this.Ui.Output(color.Blue(zkzone.Name()))

	zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
		if !patternMatched(zkcluster.Name(), clusterPattern) {
			return
		}

		this.Ui.Output(strings.Repeat(" ", 4) + zkcluster.Name())
		consumerGroups := zkcluster.ConsumerGroups()
		for group, consumers := range consumerGroups {
			if len(consumers) > 0 {
				// this consumer group is online
				continue
			}

			if !patternMatched(group, this.groupPattern) {
				continue
			}

			if !strings.HasPrefix(group, "console-consumer-") {
				path := zkcluster.ConsumerGroupOffsetPath(group)
				_, _, err := zkzone.Conn().Children(path)
				if err == nil {
					this.Ui.Warn(fmt.Sprintf("%s not empty, unsafe to cleanup", path))
					continue
				}

				if err != gozk.ErrNoNode {
					// should never happen
					swallow(err)
				}
			}

			// have no offsets, safe to delete
			if this.confirmYes {
				yes, err := this.Ui.Ask(fmt.Sprintf("confirm to remove cluster[%s] consumer group: %s? [Y/n]",
					zkcluster.Name(), group))
				swallow(err)

				if strings.ToLower(yes) == "n" {
					this.Ui.Info(fmt.Sprintf("%s skipped", group))
					continue
				}
			} else {
				yes, err := this.Ui.Ask(fmt.Sprintf("confirm to remove cluster[%s] consumer group: %s? [y/N]",
					zkcluster.Name(), group))
				swallow(err)

				if strings.ToLower(yes) != "y" {
					this.Ui.Info(fmt.Sprintf("%s skipped", group))
					continue
				}
			}

			// do delete this consumer group
			zkzone.DeleteRecursive(zkcluster.ConsumerGroupRoot(group))
			this.Ui.Info(fmt.Sprintf("%s deleted", group))
		}
	})
}

func (this *Consumers) printConsumersByGroupTable(zkzone *zk.ZkZone, clusterPattern string) {
	lines := make([]string, 0)
	header := "Zone|Cluster|M|Host|ConsumerGroup|Topic/Partition|Offset|Uptime"
	lines = append(lines, header)

	zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
		groupTopicsMap := make(map[string]map[string]struct{}) // group:sub topics

		if !patternMatched(zkcluster.Name(), clusterPattern) {
			return
		}

		consumerGroups := zkcluster.ConsumerGroups()
		sortedGroups := make([]string, 0, len(consumerGroups))
		for group, _ := range consumerGroups {
			if !patternMatched(group, this.groupPattern) {
				continue
			}

			sortedGroups = append(sortedGroups, group)
		}

		sort.Strings(sortedGroups)
		for _, group := range sortedGroups {
			consumers := consumerGroups[group]
			if this.onlineOnly && len(consumers) == 0 {
				continue
			}

			if len(consumers) > 0 {
				// sort by host
				sortedIds := make([]string, 0)
				consumersMap := make(map[string]*zk.ConsumerZnode)
				for _, c := range consumers {
					sortedIds = append(sortedIds, c.Id)
					consumersMap[c.Id] = c
				}
				sort.Strings(sortedIds)

				for _, consumerId := range sortedIds {
					c := consumersMap[consumerId]
					for topic, _ := range c.Subscription {
						if !patternMatched(topic, this.topicPattern) {
							continue
						}

						if len(groupTopicsMap[group]) == 0 {
							groupTopicsMap[group] = make(map[string]struct{}, 5)
						}
						groupTopicsMap[group][topic] = struct{}{}

						ownerByPartition := zkcluster.OwnersOfGroupByTopic(group, topic)

						partitionsWithOffset := make(map[string]struct{})
						for _, offset := range this.displayGroupOffsets(zkcluster, group, topic, false) {

							onlineSymbol := "◉"
							isOwner := false
							if ownerByPartition[offset.partitionId] == consumerId {
								onlineSymbol += "*" // owned by this consumer
								isOwner = true
							}
							if this.ownerOnly && !isOwner {
								continue
							}

							partitionsWithOffset[offset.partitionId] = struct{}{}

							lines = append(lines,
								fmt.Sprintf("%s|%s|%s|%s|%s|%s|%s|%s",
									zkzone.Name(), zkcluster.Name(),
									onlineSymbol,
									c.Host(),
									group+"@"+c.Id[len(c.Id)-12:],
									fmt.Sprintf("%s/%s", offset.topic, offset.partitionId),
									offset.offset,
									gofmt.PrettySince(c.Uptime())))
						}

						for partitionId, _ := range ownerByPartition {
							if _, present := partitionsWithOffset[partitionId]; !present {
								// this consumer is owner online, but has no offset
								onlineSymbol := "◉"
								isOwner := false
								if ownerByPartition[partitionId] == consumerId {
									onlineSymbol += "*"
									isOwner = true
								}
								if this.ownerOnly && !isOwner {
									continue
								}

								lines = append(lines,
									fmt.Sprintf("%s|%s|%s|%s|%s|%s|?|%s",
										zkzone.Name(), zkcluster.Name(),
										onlineSymbol,
										c.Host(),
										group+"@"+c.Id[len(c.Id)-12:],
										fmt.Sprintf("%s/%s", topic, partitionId),
										gofmt.PrettySince(c.Uptime())))
//.........这里部分代码省略.........