Golang transformer.GroupRecords函數代碼示例

本文整理匯總了Golang中github.com/sburnett/transformer.GroupRecords函數的典型用法代碼示例。如果您正苦於以下問題：Golang GroupRecords函數的具體用法？Golang GroupRecords怎麽用？Golang GroupRecords使用的例子？那麽, 這裏精選的函數代碼示例或許可以為您提供幫助。

在下文中一共展示了GroupRecords函數的15個代碼示例，這些例子默認根據受歡迎程度排序。您可以為喜歡或者感覺有用的代碼點讚，您的評價將有助於係統推薦出更棒的Golang代碼示例。

示例1: joinMacWithFlowId

func joinMacWithFlowId(inputChan, outputChan chan *store.Record) {
	var (
		session SessionKey
		localIp []byte
	)
	grouper := transformer.GroupRecords(inputChan, &session, &localIp)
	for grouper.NextGroup() {
		var macAddress []byte
		for grouper.NextRecord() {
			record := grouper.Read()
			switch record.DatabaseIndex {
			case 0:
				lex.DecodeOrDie(record.Value, &macAddress)
			case 1:
				if macAddress != nil {
					var (
						remoteIp       []byte
						sequenceNumber int32
						timestamp      int64
						flowId         int32
					)
					lex.DecodeOrDie(record.Key, &sequenceNumber, &remoteIp, &timestamp, &flowId)
					outputChan <- &store.Record{
						Key: lex.EncodeOrDie(&session, macAddress, remoteIp, timestamp, int64(math.MaxInt64), sequenceNumber, flowId),
					}
				}
			}
		}
	}
}

開發者ID:sburnett，項目名稱:bismark-passive-server-go，代碼行數:30，代碼來源:bytesperdomain.go

示例2: joinDomainsWithWhitelist

func joinDomainsWithWhitelist(inputChan, outputChan chan *store.Record) {
	var session SessionKey
	grouper := transformer.GroupRecords(inputChan, &session)
	for grouper.NextGroup() {
		var whitelist []string
		for grouper.NextRecord() {
			record := grouper.Read()

			switch record.DatabaseIndex {
			case 0:
				lex.DecodeOrDie(record.Value, &whitelist)
				sort.Sort(sort.StringSlice(whitelist))
			case 1:
				if whitelist == nil {
					continue
				}
				var domain string
				remainder := lex.DecodeOrDie(record.Key, &domain)
				for i := 0; i < len(domain); i++ {
					if i > 0 && domain[i-1] != '.' {
						continue
					}
					idx := sort.SearchStrings(whitelist, domain[i:])
					if idx >= len(whitelist) || whitelist[idx] != domain[i:] {
						continue
					}
					outputChan <- &store.Record{
						Key: lex.Concatenate(grouper.CurrentGroupPrefix, remainder, lex.EncodeOrDie(whitelist[idx])),
					}
				}
			}
		}
	}
}

開發者ID:sburnett，項目名稱:bismark-passive-server-go，代碼行數:34，代碼來源:bytesperdomain.go

示例3: summarizeFilesystemUsage

func summarizeFilesystemUsage(inputChan, outputChan chan *store.Record) {
	var filesystem string
	var timestamp int64
	grouper := transformer.GroupRecords(inputChan, &filesystem, &timestamp)
	for grouper.NextGroup() {
		usage := make(map[string]int64)
		for grouper.NextRecord() {
			record := grouper.Read()
			var node string
			lex.DecodeOrDie(record.Key, &node)
			var used int64
			lex.DecodeOrDie(record.Value, &used)

			if used > usage[node] {
				usage[node] = used
			}
		}
		for node, used := range usage {
			outputChan <- &store.Record{
				Key:   lex.EncodeOrDie(filesystem, timestamp, node),
				Value: lex.EncodeOrDie(used),
			}
		}
	}
}

開發者ID:sburnett，項目名稱:bismark-tools，代碼行數:25，代碼來源:summarize.go

示例4: joinAddressIdsWithMacAddresses

func joinAddressIdsWithMacAddresses(inputChan, outputChan chan *store.Record) {
	var (
		session   SessionKey
		addressId int32
	)
	grouper := transformer.GroupRecords(inputChan, &session, &addressId)
	for grouper.NextGroup() {
		var macAddress []byte
		for grouper.NextRecord() {
			record := grouper.Read()
			switch record.DatabaseIndex {
			case 0:
				macAddress = record.Value
			case 1:
				if macAddress != nil {
					var unusedSequenceNumber int32
					remainder := lex.DecodeOrDie(record.Key, &unusedSequenceNumber)
					outputChan <- &store.Record{
						Key: lex.Concatenate(lex.EncodeOrDie(&session), macAddress, remainder),
					}
				}
			}
		}
	}
}

開發者ID:sburnett，項目名稱:bismark-passive-server-go，代碼行數:25，代碼來源:bytesperdomain.go

示例5: flattenLookupsToNodeMacAndTimestamp

func flattenLookupsToNodeMacAndTimestamp(inputChan, outputChan chan *store.Record) {
	var nodeId, macAddress, domain string
	grouper := transformer.GroupRecords(inputChan, &nodeId, &macAddress, &domain)
	for grouper.NextGroup() {
		totalCounts := make(map[int64]int64)
		for grouper.NextRecord() {
			record := grouper.Read()
			var (
				anonymizationContext string
				sessionId            int64
				sequenceNumber       int32
			)
			lex.DecodeOrDie(record.Key, &anonymizationContext, &sessionId, &sequenceNumber)
			var count int64
			lex.DecodeOrDie(record.Value, &count)
			timestamp := truncateTimestampToHour(sessionId + convertSecondsToMicroseconds(30)*int64(sequenceNumber))
			totalCounts[timestamp] += count
		}
		for timestamp, totalCount := range totalCounts {
			outputChan <- &store.Record{
				Key:   lex.EncodeOrDie(nodeId, macAddress, domain, timestamp),
				Value: lex.EncodeOrDie(totalCount),
			}
		}
	}
}

開發者ID:sburnett，項目名稱:bismark-passive-server-go，代碼行數:26，代碼來源:lookupsperdevice.go

示例6: joinMacWithLookups

func joinMacWithLookups(inputChan, outputChan chan *store.Record) {
	var (
		session   SessionKey
		addressId int32
	)
	grouper := transformer.GroupRecords(inputChan, &session, &addressId)
	for grouper.NextGroup() {
		var macAddress []byte
		for grouper.NextRecord() {
			record := grouper.Read()
			switch record.DatabaseIndex {
			case 0:
				lex.DecodeOrDie(record.Value, &macAddress)
			case 1:
				if macAddress != nil {
					var (
						sequenceNumber int32
						domain         string
					)
					lex.DecodeOrDie(record.Key, &sequenceNumber, &domain)
					outputChan <- &store.Record{
						Key:   lex.EncodeOrDie(session.NodeId, macAddress, domain, session.AnonymizationContext, session.SessionId, sequenceNumber),
						Value: record.Value,
					}
				}
			}
		}
	}
}

開發者ID:sburnett，項目名稱:bismark-passive-server-go，代碼行數:29，代碼來源:lookupsperdevice.go

示例7: aggregateStatisticsReducer

func aggregateStatisticsReducer(inputChan, outputChan chan *store.Record) {
	var nodeId []byte
	grouper := transformer.GroupRecords(inputChan, &nodeId)
	for grouper.NextGroup() {
		aggregateStatistics := newAggregateStatistics()
		for grouper.NextRecord() {
			record := grouper.Read()
			var statistics AggregateStatistics
			if err := proto.Unmarshal(record.Value, &statistics); err != nil {
				panic(err)
			}
			*aggregateStatistics.Traces += *statistics.Traces
			*aggregateStatistics.Packets += *statistics.Packets
			*aggregateStatistics.PacketSeriesDropped += *statistics.PacketSeriesDropped
			*aggregateStatistics.PcapDropped += *statistics.PcapDropped
			*aggregateStatistics.InterfaceDropped += *statistics.InterfaceDropped
			*aggregateStatistics.Flows += *statistics.Flows
			*aggregateStatistics.DroppedFlows += *statistics.DroppedFlows
			*aggregateStatistics.Bytes += *statistics.Bytes
		}
		encodedStatistics, err := proto.Marshal(aggregateStatistics)
		if err != nil {
			panic(err)
		}
		outputChan <- &store.Record{
			Key:   lex.EncodeOrDie(nodeId),
			Value: encodedStatistics,
		}
	}
}

開發者ID:sburnett，項目名稱:bismark-passive-server-go，代碼行數:30，代碼來源:statistics.go

示例8: inferReboots

func inferReboots(inputChan, outputChan chan *store.Record) {
	var node string
	grouper := transformer.GroupRecords(inputChan, &node)
	for grouper.NextGroup() {
		lastUptime := int64(-1)
		maxReboot := int64(-1)
		for grouper.NextRecord() {
			record := grouper.Read()
			var timestamp int64
			lex.DecodeOrDie(record.Key, &timestamp)
			var uptime int64
			lex.DecodeOrDie(record.Value, &uptime)

			if lastUptime >= 0 && lastUptime > uptime {
				if maxReboot > -1 {
					outputChan <- &store.Record{
						Key: lex.EncodeOrDie(node, maxReboot),
					}
				}
				maxReboot = int64(-1)
			}
			reboot := timestamp - uptime
			if maxReboot < reboot {
				maxReboot = reboot
			}
			lastUptime = uptime
		}

		if maxReboot > -1 {
			outputChan <- &store.Record{
				Key: lex.EncodeOrDie(node, maxReboot),
			}
		}
	}
}

開發者ID:sburnett，項目名稱:bismark-tools，代碼行數:35，代碼來源:reboots.go

示例9: joinMacAndSizes

func joinMacAndSizes(inputChan, outputChan chan *store.Record) {
	var session SessionKey
	var flowId int32
	grouper := transformer.GroupRecords(inputChan, &session, &flowId)
	for grouper.NextGroup() {
		var currentMacAddresses [][]byte
		for grouper.NextRecord() {
			record := grouper.Read()
			if record.DatabaseIndex == 0 {
				lex.DecodeOrDie(record.Value, &currentMacAddresses)
				continue
			}
			if currentMacAddresses == nil {
				continue
			}

			var sequenceNumber int32
			lex.DecodeOrDie(record.Key, &sequenceNumber)
			var timestamps, sizes []int64
			lex.DecodeOrDie(record.Value, &timestamps, &sizes)
			if len(timestamps) != len(sizes) {
				panic(fmt.Errorf("timestamps and sizes must be the same size"))
			}

			for _, currentMacAddress := range currentMacAddresses {
				for idx, timestamp := range timestamps {
					outputChan <- &store.Record{
						Key:   lex.EncodeOrDie(&session, currentMacAddress, timestamp, flowId, sequenceNumber),
						Value: lex.EncodeOrDie(sizes[idx]),
					}
				}
			}
		}
	}
}

開發者ID:sburnett，項目名稱:bismark-passive-server-go，代碼行數:35，代碼來源:bytesperdevice.go

示例10: summarizeSizesByTimestamp

func summarizeSizesByTimestamp(inputChan, outputChan chan *store.Record) {
	quantileComputer := NewQuantileSample(101)
	quantilesToKeep := []int{0, 1, 5, 10, 25, 50, 75, 90, 95, 99, 100}

	var experiment, node string
	var timestamp int64
	grouper := transformer.GroupRecords(inputChan, &experiment, &node, &timestamp)
	for grouper.NextGroup() {
		for grouper.NextRecord() {
			record := grouper.Read()
			var statsValue StatsValue
			lex.DecodeOrDie(record.Value, &statsValue)
			quantileComputer.Append(statsValue.Size)
		}

		count := int64(quantileComputer.Count())
		quantiles := quantileComputer.Quantiles()
		quantileComputer.Reset()

		statistics := []interface{}{count}
		for idx := range quantilesToKeep {
			statistics = append(statistics, quantiles[idx])
		}

		outputChan <- &store.Record{
			Key:   lex.EncodeOrDie(experiment, node, timestamp),
			Value: lex.EncodeOrDie(statistics...),
		}
	}
}

開發者ID:sburnett，項目名稱:bismark-tools，代碼行數:30，代碼來源:summary.go

示例11: summarizeInterarrivalTimes

func summarizeInterarrivalTimes(inputChan, outputChan chan *store.Record) {
	quantileComputer := NewQuantileSample(21)

	var experiment, node string
	grouper := transformer.GroupRecords(inputChan, &experiment, &node)
	for grouper.NextGroup() {
		for grouper.NextRecord() {
			record := grouper.Read()
			var timestamp int64
			lex.DecodeOrDie(record.Key, &timestamp)
			quantileComputer.Append(timestamp)
		}

		count := int64(quantileComputer.Count())
		quantiles := quantileComputer.Quantiles()
		statistics := []interface{}{count}
		for _, q := range quantiles {
			statistics = append(statistics, q)
		}
		quantileComputer.Reset()

		outputChan <- &store.Record{
			Key:   lex.EncodeOrDie(experiment, node),
			Value: lex.EncodeOrDie(statistics...),
		}
	}
}

開發者ID:sburnett，項目名稱:bismark-tools，代碼行數:27，代碼來源:summary.go

示例12: joinMacAndFlowId

func joinMacAndFlowId(inputChan, outputChan chan *store.Record) {
	var session SessionKey
	var ipAddress []byte
	grouper := transformer.GroupRecords(inputChan, &session, &ipAddress)
	for grouper.NextGroup() {
		var currentMacAddress []byte
		for grouper.NextRecord() {
			record := grouper.Read()
			if record.DatabaseIndex == 0 {
				currentMacAddress = record.Value
				continue
			}
			if currentMacAddress == nil {
				continue
			}
			var sequenceNumber int32
			lex.DecodeOrDie(record.Key, &sequenceNumber)
			var flowIds []int32
			lex.DecodeOrDie(record.Value, &flowIds)
			for _, flowId := range flowIds {
				outputChan <- &store.Record{
					Key: lex.Concatenate(lex.EncodeOrDie(&session, flowId, sequenceNumber), currentMacAddress),
				}
			}
		}
	}
}

開發者ID:sburnett，項目名稱:bismark-passive-server-go，代碼行數:27，代碼來源:bytesperdevice.go

示例13: joinDomainsWithSizes

func joinDomainsWithSizes(inputChan, outputChan chan *store.Record) {
	var (
		session SessionKey
		flowId  int32
	)
	grouper := transformer.GroupRecords(inputChan, &session, &flowId)
	for grouper.NextGroup() {
		var domains, macAddresses [][]byte
		for grouper.NextRecord() {
			record := grouper.Read()

			switch record.DatabaseIndex {
			case 0:
				lex.DecodeOrDie(record.Value, &domains, &macAddresses)
			case 1:
				if domains != nil && macAddresses != nil {
					var (
						sequenceNumber int32
						timestamp      int64
					)
					lex.DecodeOrDie(record.Key, &sequenceNumber, &timestamp)
					for idx, domain := range domains {
						outputChan <- &store.Record{
							Key:   lex.EncodeOrDie(session.NodeId, domain, timestamp, macAddresses[idx], session.AnonymizationContext, session.SessionId, flowId, sequenceNumber),
							Value: record.Value,
						}
					}
				}
			}
		}
	}
}

開發者ID:sburnett，項目名稱:bismark-passive-server-go，代碼行數:32，代碼來源:bytesperdomain.go

示例14: aggregateStatisticsReduceBySession

func aggregateStatisticsReduceBySession(inputChan, outputChan chan *store.Record) {
	var session SessionKey
	grouper := transformer.GroupRecords(inputChan, &session)
	for grouper.NextGroup() {
		aggregateStatistics := newAggregateStatistics()
		var pcapDropped, interfaceDropped int64
		var lastPcapDropped, lastInterfaceDropped int64
		var pcapDroppedBaseline, interfaceDroppedBaseline int64
		for grouper.NextRecord() {
			record := grouper.Read()
			var statistics AggregateStatistics
			if err := proto.Unmarshal(record.Value, &statistics); err != nil {
				panic(err)
			}

			if lastPcapDropped > *statistics.PcapDropped {
				pcapDroppedBaseline += math.MaxUint32
				pcapDropped = 0
			}
			lastPcapDropped = *statistics.PcapDropped
			pcapDropped = maxInt64(pcapDropped, *statistics.PcapDropped)
			if lastInterfaceDropped > *statistics.InterfaceDropped {
				interfaceDroppedBaseline += math.MaxUint32
				interfaceDropped = 0
			}
			lastInterfaceDropped = *statistics.InterfaceDropped
			interfaceDropped = maxInt64(interfaceDropped, *statistics.InterfaceDropped)

			*aggregateStatistics.Traces += *statistics.Traces
			*aggregateStatistics.Packets += *statistics.Packets
			*aggregateStatistics.PacketSeriesDropped += *statistics.PacketSeriesDropped
			*aggregateStatistics.Flows += *statistics.Flows
			*aggregateStatistics.DroppedFlows += *statistics.DroppedFlows
			*aggregateStatistics.Bytes += *statistics.Bytes
		}

		*aggregateStatistics.PcapDropped = pcapDroppedBaseline + pcapDropped
		*aggregateStatistics.InterfaceDropped = interfaceDroppedBaseline + interfaceDropped

		encodedStatistics, err := proto.Marshal(aggregateStatistics)
		if err != nil {
			panic(err)
		}
		outputChan <- &store.Record{
			Key:   lex.EncodeOrDie(&session),
			Value: encodedStatistics,
		}
	}
}

開發者ID:sburnett，項目名稱:bismark-passive-server-go，代碼行數:49，代碼來源:statistics.go

示例15: flattenLookupsToNodeAndMac

func flattenLookupsToNodeAndMac(inputChan, outputChan chan *store.Record) {
	var nodeId, macAddress, domain string
	grouper := transformer.GroupRecords(inputChan, &nodeId, &macAddress, &domain)
	for grouper.NextGroup() {
		var totalCount int64
		for grouper.NextRecord() {
			record := grouper.Read()
			var count int64
			lex.DecodeOrDie(record.Value, &count)
			totalCount += count
		}
		outputChan <- &store.Record{
			Key:   lex.EncodeOrDie(nodeId, macAddress, domain),
			Value: lex.EncodeOrDie(totalCount),
		}
	}
}

開發者ID:sburnett，項目名稱:bismark-passive-server-go，代碼行數:17，代碼來源:lookupsperdevice.go

注：本文中的github.com/sburnett/transformer.GroupRecords函數示例由純淨天空整理自Github/MSDocs等開源代碼及文檔管理平台，相關代碼片段篩選自各路編程大神貢獻的開源項目，源碼版權歸原作者所有，傳播和使用請參考對應項目的License；未經允許，請勿轉載。