Golang semaphore.NewSemaphore函數代碼示例

// TODO 添加對發送任務的控製,比如stop等
func startSendTasks() {
	cfg := g.Config()
	// init semaphore
	judgeConcurrent := cfg.Judge.MaxIdle / 2
	graphConcurrent := cfg.Graph.MaxIdle / 2
	if judgeConcurrent < 1 {
		judgeConcurrent = 1
	}
	if graphConcurrent < 1 {
		graphConcurrent = 1
	}
	semaSendToJudge = nsema.NewSemaphore(judgeConcurrent)
	semaSendToGraph = nsema.NewSemaphore(graphConcurrent)
	semaSendToGraphMigrating = nsema.NewSemaphore(graphConcurrent)

	// init send go-routines
	for node, _ := range cfg.Judge.Cluster {
		queue := JudgeQueues[node]
		go forward2JudgeTask(queue, node, judgeConcurrent)
	}

	for node, _ := range cfg.Graph.Cluster {
		queue := GraphQueues[node]
		go forward2GraphTask(queue, node, graphConcurrent)
	}

	if cfg.Graph.Migrating {
		for node, _ := range cfg.Graph.ClusterMigrating {
			queue := GraphMigratingQueues[node]
			go forward2GraphMigratingTask(queue, node, graphConcurrent)
		}
	}
}

func SWifMetricToTransfer() {
	log.Println("start SWifMetricToTransfer")
	sema := nsema.NewSemaphore(10)

	for {
		items := IfstatsQueue.PopBackBy(5000)

		count := len(items)
		if count == 0 {
			time.Sleep(DefaultSendTaskSleepInterval)
			continue
		}

		mvsSend := make([]*model.MetricValue, count)
		for i := 0; i < count; i++ {
			mvsSend[i] = items[i].(*model.MetricValue)
		}

		//	同步Call + 有限並發 進行發送
		sema.Acquire()
		go func(mvsend []*model.MetricValue) {
			defer sema.Release()
			g.SendToTransfer(mvsend)
		}(mvsSend)
	}

}

func collectDataOnce() int {
	keys := config.Keys()
	keysLen := len(keys)

	// 並發+同步控製
	cfg := g.Config().Collector
	concurrent := int(cfg.Concurrent)
	if concurrent < 1 || concurrent > 50 {
		concurrent = 10
	}
	sema := tsema.NewSemaphore(concurrent)

	batch := int(cfg.Batch)
	if batch < 100 || batch > 1000 {
		batch = 200 //batch不能太小, 否則channel將會很大
	}

	batchCnt := (keysLen + batch - 1) / batch
	rch := make(chan int, batchCnt+1)

	i := 0
	for i < keysLen {
		leftLen := keysLen - i
		fetchSize := batch // 每次處理batch個配置
		if leftLen < fetchSize {
			fetchSize = leftLen
		}
		fetchKeys := keys[i : i+fetchSize]

		// 並發collect數據
		sema.Acquire()
		go func(keys []string, keySize int) {
			defer sema.Release()
			size, _ := fetchItemsAndStore(keys, keySize)
			rch <- size
		}(fetchKeys, fetchSize)

		i += fetchSize
	}

	collectCnt := 0
	for i := 0; i < batchCnt; i++ {
		select {
		case cnt := <-rch:
			collectCnt += cnt
		}
	}

	return collectCnt
}

// Judge定時任務, 將 Judge發送緩存中的數據 通過rpc連接池 發送到Judge
func forward2JudgeTask(Q *list.SafeLinkedListLimited, node string, concurrent int) {
	batch := g.Config().Judge.Batch // 一次發送,最多batch條數據
	addr := g.Config().Judge.Cluster[node]
	sema := nsema.NewSemaphore(concurrent)

	for {
		items := Q.PopBack(batch)
		count := len(items)
		if count == 0 {
			time.Sleep(DefaultSendTaskSleepInterval)
			continue
		}

		judgeItems := make([]*cmodel.JudgeItem, count)
		for i := 0; i < count; i++ {
			judgeItems[i] = items[i].(*cmodel.JudgeItem)
		}

		//	同步Call + 有限並發 進行發送
		sema.Acquire()
		go func(addr string, judgeItems []*cmodel.JudgeItem, count int) {
			defer sema.Release()

			resp := &cmodel.SimpleRpcResponse{}
			var err error
			sendOk := false
			for i := 0; i < 3; i++ { //最多重試3次
				err = JudgeConnPools.Call(addr, "Judge.Send", judgeItems, resp)
				if err == nil {
					sendOk = true
					break
				}
				time.Sleep(time.Millisecond * 10)
			}

			if !sendOk {
				log.Printf("send judge %s fail: %v", addr, err)
				// statistics
				proc.SendToJudgeFailCnt.IncrBy(int64(count))
			} else {
				// statistics
				proc.SendToJudgeCnt.IncrBy(int64(count))
			}
		}(addr, judgeItems, count)
	}
}

// Graph定時任務, 進行數據遷移時的 數據冗餘發送
func forward2GraphMigratingTask(Q *list.SafeLinkedListLimited, node string, concurrent int) {
	batch := g.Config().Graph.Batch // 一次發送,最多batch條數據
	addr := g.Config().Graph.ClusterMigrating[node]
	sema := nsema.NewSemaphore(concurrent)

	for {
		items := Q.PopBack(batch)
		count := len(items)
		if count == 0 {
			time.Sleep(DefaultSendTaskSleepInterval)
			continue
		}

		graphItems := make([]*cmodel.GraphItem, count)
		for i := 0; i < count; i++ {
			graphItems[i] = items[i].(*cmodel.GraphItem)
		}

		sema.Acquire()
		go func(addr string, graphItems []*cmodel.GraphItem, count int) {
			defer sema.Release()

			resp := &cmodel.SimpleRpcResponse{}
			var err error
			sendOk := false
			for i := 0; i < 3; i++ { //最多重試3次
				err = GraphMigratingConnPools.Call(addr, "Graph.Send", graphItems, resp)
				if err == nil {
					sendOk = true
					break
				}
				time.Sleep(time.Millisecond * 10) //發送失敗了,睡10ms
			}

			if !sendOk {
				log.Printf("send to graph migrating %s fail: %v", addr, err)
				// statistics
				proc.SendToGraphMigratingFailCnt.IncrBy(int64(count))
			} else {
				// statistics
				proc.SendToGraphMigratingCnt.IncrBy(int64(count))
			}
		}(addr, graphItems, count)
	}
}

// Tsdb定時任務, 將數據通過api發送到tsdb
func forward2TsdbTask(concurrent int) {
	batch := g.Config().Tsdb.Batch // 一次發送,最多batch條數據
	retry := g.Config().Tsdb.MaxRetry
	sema := nsema.NewSemaphore(concurrent)

	for {
		items := TsdbQueue.PopBackBy(batch)
		if len(items) == 0 {
			time.Sleep(DefaultSendTaskSleepInterval)
			continue
		}
		//  同步Call + 有限並發 進行發送
		sema.Acquire()
		go func(itemList []interface{}) {
			defer sema.Release()

			var tsdbBuffer bytes.Buffer
			for i := 0; i < len(itemList); i++ {
				tsdbItem := itemList[i].(*cmodel.TsdbItem)
				tsdbBuffer.WriteString(tsdbItem.TsdbString())
				tsdbBuffer.WriteString("\n")
			}

			var err error
			for i := 0; i < retry; i++ {
				err = TsdbConnPoolHelper.Send(tsdbBuffer.Bytes())
				if err == nil {
					proc.SendToTsdbCnt.IncrBy(int64(len(itemList)))
					break
				}
				time.Sleep(100 * time.Millisecond)
			}

			if err != nil {
				proc.SendToTsdbFailCnt.IncrBy(int64(len(itemList)))
				log.Println(err)
				return
			}
		}(items)
	}
}

// Tsdb定時任務, 將 Tsdb發送緩存中的數據 通過api連接池 發送到Tsdb
// 單個Cluster配置多個influxdb地址，修改為並發發送。
func forward2InfluxdbTask(Q *list.SafeListLimited, node string, concurrent int) {
	cfg := g.Config()
	batch := cfg.Influxdb.Batch // 一次發送,最多batch條數據
	sema := nsema.NewSemaphore(concurrent * len(cfg.Influxdb.Cluster2[node].Addrs))
	retry := cfg.Influxdb.MaxRetry

	for {
		items := Q.PopBackBy(batch)
		count := len(items)
		if count == 0 {
			time.Sleep(DefaultSendTaskSleepInterval)
			continue
		}
		pts := make([]*client.Point, count)
		for i := 0; i < count; i++ {
			pts[i] = items[i].(*client.Point)
		}
		for _, addr := range cfg.Influxdb.Cluster2[node].Addrs {
			sema.Acquire()
			go coreSend2Influxdb(addr, sema, retry, pts)
		}

	}
}

	"net/http"
	"time"

	cmodel "github.com/open-falcon/common/model"
	tsema "github.com/toolkits/concurrent/semaphore"
	"github.com/toolkits/container/nmap"
	thttpclient "github.com/toolkits/http/httpclient"
	ttime "github.com/toolkits/time"

	"github.com/open-falcon/nodata/config"
	"github.com/open-falcon/nodata/g"
)

var (
	MockMap = nmap.NewSafeMap()
	sema    = tsema.NewSemaphore(1)
)

func Start() {
	if !g.Config().Sender.Enabled {
		log.Println("sender.Start warning, not enabled")
		return
	}
	startGaussCron()
	log.Println("sender.Start ok")
}

func AddMock(key string, endpoint string, metric string, tags string, ts int64, dstype string, step int64, value interface{}) {
	item := &cmodel.JsonMetaData{metric, endpoint, ts, step, value, dstype, tags}
	MockMap.Put(key, item)
}

	cutils "github.com/open-falcon/common/utils"
	"github.com/open-falcon/graph/g"
	proc "github.com/open-falcon/graph/proc"
	nsema "github.com/toolkits/concurrent/semaphore"
	ntime "github.com/toolkits/time"
	"log"
	"time"
)

const (
	DefaultUpdateStepInSec     = 2 * 24 * 3600 //更新步長,一定不能大於刪除步長. 兩天內的數據,都可以用來建立索引
	ConcurrentOfUpdateIndexAll = 1
)

var (
	semaIndexUpdateAllTask = nsema.NewSemaphore(ConcurrentOfUpdateIndexAll) //全量同步任務 並發控製器
	semaIndexUpdateAll     = nsema.NewSemaphore(4)                          // 索引全量更新時的mysql操作並發控製
)

// 更新一條監控數據對應的索引. 用於手動添加索引,一般情況下不會使用
func UpdateIndexOne(endpoint string, metric string, tags map[string]string, dstype string, step int) error {
	log.Println("1")
	itemDemo := &cmodel.GraphItem{
		Endpoint: endpoint,
		Metric:   metric,
		Tags:     tags,
		DsType:   dstype,
		Step:     step,
	}
	md5 := itemDemo.Checksum()
	uuid := itemDemo.UUID()

	nsema "github.com/toolkits/concurrent/semaphore"
	ntime "github.com/toolkits/time"

	cmodel "github.com/open-falcon/common/model"
	cutils "github.com/open-falcon/common/utils"
	"github.com/open-falcon/graph/g"
	proc "github.com/open-falcon/graph/proc"
)

const (
	IndexUpdateIncrTaskSleepInterval = time.Duration(1) * time.Second // 增量更新間隔時間, 默認30s
)

var (
	semaUpdateIndexIncr = nsema.NewSemaphore(2) // 索引增量更新時操作mysql的並發控製
)

// 啟動索引的 異步、增量更新 任務
func StartIndexUpdateIncrTask() {
	for {
		time.Sleep(IndexUpdateIncrTaskSleepInterval)
		startTs := time.Now().Unix()
		cnt := updateIndexIncr()
		endTs := time.Now().Unix()
		// statistics
		proc.IndexUpdateIncrCnt.SetCnt(int64(cnt))
		proc.IndexUpdateIncr.Incr()
		proc.IndexUpdateIncr.PutOther("lastStartTs", ntime.FormatTs(startTs))
		proc.IndexUpdateIncr.PutOther("lastTimeConsumingInSec", endTs-startTs)
	}

import (
	Mdb "github.com/open-falcon/common/db"
	"github.com/open-falcon/task/proc"
	TSemaphore "github.com/toolkits/concurrent/semaphore"
	cron "github.com/toolkits/cron"
	"log"
	"time"
)

const (
	indexDeleteCronSpec = "0 0 2 ? * 6" // 每周6晚上22:00執行一次
	deteleStepInSec     = 7 * 24 * 3600 // 索引的最大生存周期, sec
)

var (
	semaIndexDelete = TSemaphore.NewSemaphore(1)
	indexDeleteCron = cron.New()
)

// 啟動 索引全量更新 定時任務
func StartIndexDeleteTask() {
	indexDeleteCron.AddFunc(indexDeleteCronSpec, func() {
		DeleteIndex()
	})
	indexDeleteCron.Start()
}

func StopIndexDeleteTask() {
	indexDeleteCron.Stop()
}

	nsema "github.com/toolkits/concurrent/semaphore"
	nmap "github.com/toolkits/container/nmap"
	ncron "github.com/toolkits/cron"
	nhttpclient "github.com/toolkits/http/httpclient"
	ntime "github.com/toolkits/time"
	"io/ioutil"
	"log"
	"net/http"
	"strings"
	"sync"
	"time"
)

var (
	monitorCron = ncron.New()
	sema        = nsema.NewSemaphore(1)
	statusCache = nmap.NewSafeMap()
	alarmCache  = nmap.NewSafeMap()
	cronSpec    = "30 * * * * ?"
)

func Start() {
	if !g.Config().Monitor.Enabled {
		log.Println("monitor.Start, not enable")
		return
	}
	//
	monitorCron.AddFunc(cronSpec, func() {
		monitor()
	})
	monitorCron.Start()

func forward2TransferTask(Q *nlist.SafeListLimited, concurrent int32) {
	cfg := g.Config()
	batch := int(cfg.Transfer.Batch)
	maxConns := int64(cfg.Transfer.MaxConns)
	retry := int(cfg.Transfer.Retry)
	if retry < 1 {
		retry = 1
	}

	sema := nsema.NewSemaphore(int(concurrent))
	transNum := len(TransferHostnames)

	for {
		items := Q.PopBackBy(batch)
		count := len(items)
		if count == 0 {
			time.Sleep(time.Millisecond * 50)
			continue
		}

		transItems := make([]*cmodel.MetricValue, count)
		for i := 0; i < count; i++ {
			transItems[i] = convert(items[i].(*cmodel.MetaData))
		}

		sema.Acquire()
		go func(transItems []*cmodel.MetricValue, count int) {
			defer sema.Release()
			var err error

			// 隨機遍曆transfer列表，直到數據發送成功 或者 遍曆完;隨機遍曆，可以緩解慢transfer
			resp := &g.TransferResp{}
			sendOk := false

			for j := 0; j < retry && !sendOk; j++ {
				rint := rand.Int()
				for i := 0; i < transNum && !sendOk; i++ {
					idx := (i + rint) % transNum
					host := TransferHostnames[idx]
					addr := TransferMap[host]

					// 過濾掉建連緩慢的host, 否則會嚴重影響發送速率
					cc := pfc.GetCounterCount(host)
					if cc >= maxConns {
						continue
					}

					pfc.Counter(host, 1)
					err = SenderConnPools.Call(addr, "Transfer.Update", transItems, resp)
					pfc.Counter(host, -1)

					if err == nil {
						sendOk = true
						// statistics
						TransferSendCnt[host].IncrBy(int64(count))
					} else {
						// statistics
						TransferSendFailCnt[host].IncrBy(int64(count))
					}
				}
			}

			// statistics
			if !sendOk {
				if cfg.Debug {
					log.Printf("send to transfer fail, connpool:%v", SenderConnPools.Proc())
				}
				pfc.Meter("SendFail", int64(count))
			} else {
				pfc.Meter("Send", int64(count))
			}
		}(transItems, count)
	}
}

	"github.com/open-falcon/task/g"
	"github.com/open-falcon/task/proc"
	sema "github.com/toolkits/concurrent/semaphore"
	cron "github.com/toolkits/cron"
	nhttpclient "github.com/toolkits/http/httpclient"
	"io/ioutil"
	"log"
	"net/http"
	"strings"
	"time"
)

var (
	collectorCron     = cron.New()
	collectorCronSpec = "0 * * * * ?"
	collectorSema     = sema.NewSemaphore(1)
	srcUrlFmt         = "http://%s/statistics/all"
	destUrl           = "http://127.0.0.1:1988/v1/push"
)

func Start() {
	if !g.Config().Collector.Enabled {
		log.Println("collector.Start, not enable")
		return
	}

	// init url
	if g.Config().Collector.DestUrl != "" {
		destUrl = g.Config().Collector.DestUrl
	}
	if g.Config().Collector.SrcUrlFmt != "" {