Golang v2.Client類代碼示例

func syncDeliveryServiceStat(sourceClient influx.Client, targetClient influx.Client, statName string, days int) {

	db := "deliveryservice_stats"
	bps, _ := influx.NewBatchPoints(influx.BatchPointsConfig{
		Database:        db,
		Precision:       "ms",
		RetentionPolicy: "monthly",
	})

	queryString := fmt.Sprintf("select time, cachegroup, cdn, deliveryservice, value from \"monthly\".\"%s\"", statName)
	if days > 0 {
		queryString += fmt.Sprintf(" where time > now() - %dd", days)
	}
	fmt.Println("queryString ", queryString)
	res, err := queryDB(sourceClient, queryString, db)
	if err != nil {
		errorMessage = fmt.Sprintf("An error occured getting %s records from sourceDb: %v\n", statName, err)
		fmt.Println(errorMessage)
		return
	}
	sourceStats := getDeliveryServiceStats(res)
	// get value from target DB
	targetRes, err := queryDB(targetClient, queryString, db)
	if err != nil {
		errorMessage = fmt.Sprintf("An error occured getting %s record from target db: %v\n", statName, err)
		fmt.Println(errorMessage)
		return
	}
	targetStats := getDeliveryServiceStats(targetRes)

	for ssKey := range sourceStats {
		ts := targetStats[ssKey]
		ss := sourceStats[ssKey]
		if ts.value > ss.value {
			//fmt.Printf("target value %v is at least equal to source value %v\n", ts.value, ss.value)
			continue //target value is bigger so leave it
		}
		statTime, _ := time.Parse(time.RFC3339, ss.t)
		tags := map[string]string{
			"cdn":             ss.cdn,
			"cachegroup":      ss.cacheGroup,
			"deliveryservice": ss.deliveryService,
		}
		fields := map[string]interface{}{
			"value": ss.value,
		}
		pt, err := influx.NewPoint(
			statName,
			tags,
			fields,
			statTime,
		)
		if err != nil {
			fmt.Printf("error adding creating point for %v...%v\n", statName, err)
			continue
		}
		bps.AddPoint(pt)
	}
	targetClient.Write(bps)
}

// InfluxDB writes interest processing time to influxDB.
//
// The data collected can be viewed with: SELECT "value" FROM :name WHERE "name" = ':interest_name'.
func InfluxDB(client influxdb.Client, db, name string, tags map[string]string) mux.Middleware {
	return func(next mux.Handler) mux.Handler {
		return mux.HandlerFunc(func(w ndn.Sender, i *ndn.Interest) {
			before := time.Now()
			next.ServeNDN(w, i)
			t := make(map[string]string)
			for k, v := range tags {
				t[k] = v
			}
			t["name"] = i.Name.String()
			pt, _ := influxdb.NewPoint(name, t, map[string]interface{}{
				"value": float64(time.Since(before)) / float64(time.Millisecond),
			}, time.Now())
			bp, _ := influxdb.NewBatchPoints(influxdb.BatchPointsConfig{
				Database: db,
			})
			bp.AddPoint(pt)

			err := client.Write(bp)
			if err != nil {
				log.Println(err)
				return
			}
		})
	}
}

func (self *Server) influxWriter(influxClient influxdb.Client, options InfluxOptions) {
	defer close(self.influxChan)
	defer influxClient.Close()

	for stat := range self.influxChan {
		tags := map[string]string{
			"id":     stat.ID.String(),
			"family": stat.ID.Family(),
			"name":   stat.SensorConfig.String(),
		}
		fields := map[string]interface{}{
			"temperature": stat.Temperature.Float64(),
		}

		// write
		point, err := influxdb.NewPoint("onewire", tags, fields, stat.Time)
		if err != nil {
			log.Printf("server.Server: influxWriter: influxdb.NewPoint: %v\n", err)
		}

		points, err := options.batchPoints()
		if err != nil {
			log.Printf("server.Server: influxWriter: newBatchPoints: %v\n", err)
			continue
		}

		points.AddPoint(point)

		if err := influxClient.Write(points); err != nil {
			log.Printf("server.Server: influxWriter: influxdb.Client %v: Write %v: %v\n", influxClient, points, err)
			continue
		}
	}
}

func processPing(c <-chan Ping) error {
	var err error
	var carbonReceiver *carbon.Carbon
	var ic influxdbclient.Client

	switch receiverTypeFlag {
	case "carbon":
		carbonReceiver, err = carbon.NewCarbon(receiverHostFlag, receiverPortFlag, receiverNoopFlag, verboseFlag)
	case "influxdb":
		if receiverDatabaseFlag == "" {
			log.Fatalln("An InfluxDB database was not specified on the command line.")
		}
		if len(receiverUsernameFlag) > 0 {
			ic, err = influxdbclient.NewHTTPClient(
				influxdbclient.HTTPConfig{
					Addr:     fmt.Sprintf("http://%v:%v", receiverHostFlag, receiverPortFlag),
					Username: receiverUsernameFlag,
					Password: receiverPasswordFlag,
				})
		} else {
			ic, err = influxdbclient.NewHTTPClient(
				influxdbclient.HTTPConfig{
					Addr: fmt.Sprintf("http://%v:%v", receiverHostFlag, receiverPortFlag),
				})
		}
	}

	if err != nil {
		log.Println("error in creating a connection, but ignoring")
		//		return err
	}
	for {
		pingResult := <-c
		if !isReceiverFullyDefined() {
			log.Printf("Receiver is not fully defined: host: %v Port %v\n.",
				receiverHostFlag, receiverPortFlag)
			continue
		}
		switch receiverTypeFlag {
		case "carbon":
			err := carbonReceiver.SendMetrics(createCarbonMetrics(pingResult))
			if err != nil {
				log.Printf("Error sending metrics to Carbon: %v.\n", err)
			}
		case "influxdb":
			ret, err := createInfluxDBMetrics(pingResult)
			if err != nil {
				log.Fatalln(err)
			}
			err = ic.Write(ret)
			if err != nil {
				log.Printf("Error writing metrics to Influxdb: %v.\n", err)
			}
		}
		if verboseFlag {
			log.Printf("Successfully published %v metrics to %v.\n", receiverTypeFlag, receiverHostFlag)
		}
	}
	return nil
}

func queryDB(client influx.Client, cmd string) (res []influx.Result, err error) {
	q := influx.Query{
		Command:  cmd,
		Database: "",
	}
	if response, err := client.Query(q); err == nil {
		if response.Error() != nil {
			return res, response.Error()
		}
		res = response.Results
	}
	return res, nil
}

func queryDB(clnt influxpackage.Client, cmd string, dbname string) (res []influxpackage.Result, err error) {
	q := influxpackage.Query{
		Command:  cmd,
		Database: dbname,
	}
	if response, err := clnt.Query(q); err == nil {
		if response.Error() != nil {
			return res, response.Error()
		}
		res = response.Results
	} else {
		return res, err
	}
	return res, nil
}

// NewInfluxDB returns a new InfluxDBHook.
func NewInfluxDB(config *Config, clients ...influxdb.Client) (hook *InfluxDBHook, err error) {
	if config == nil {
		config = &Config{}
	}
	config.defaults()

	var client influxdb.Client
	if len(clients) == 0 {
		client, err = hook.newInfluxDBClient(config)
		if err != nil {
			return nil, fmt.Errorf("NewInfluxDB: Error creating InfluxDB Client, %v", err)
		}
	} else if len(clients) == 1 {
		client = clients[0]
	} else {
		return nil, fmt.Errorf("NewInfluxDB: Error creating InfluxDB Client, %d is too many influxdb clients", len(clients))
	}

	// Make sure that we can connect to InfluxDB
	_, _, err = client.Ping(5 * time.Second) // if this takes more than 5 seconds then influxdb is probably down
	if err != nil {
		return nil, fmt.Errorf("NewInfluxDB: Error connecting to InfluxDB, %v", err)
	}

	hook = &InfluxDBHook{
		client:        client,
		database:      config.Database,
		measurement:   config.Measurement,
		tagList:       config.Tags,
		batchInterval: config.BatchInterval,
		batchCount:    config.BatchCount,
		precision:     config.Precision,
	}

	err = hook.autocreateDatabase()
	if err != nil {
		return nil, err
	}
	go hook.handleBatch()

	return hook, nil
}

// dbCheck ensures the given database exists
func dbCheck(conn client.Client, database string) error {
	if len(database) == 0 {
		return fmt.Errorf("no database specified")
	}
	q := client.Query{Command: "show databases"}
	resp, err := conn.Query(q)
	if err != nil {
		return err
	}

	for _, r := range resp.Results {
		for _, s := range r.Series {
			for _, v := range s.Values {
				for _, d := range v {
					if d.(string) == database {
						return nil
					}
				}
			}
		}
	}
	return fmt.Errorf("database %s does not exist", database)
}

// NewSender returns a function that will accept datapoints to send to influxdb
func NewSender(
	config interface{},
	batch client.BatchPointsConfig,
	batchSize int,
	queueSize int,
	flush int,
	errFunc func(error),
) (Sender, error) {
	if batchSize <= 0 {
		batchSize = DefaultBatchSize
	}
	if queueSize <= 0 {
		queueSize = DefaultQueueSize
	}
	if flush <= 0 {
		flush = DefaultFlush
	}

	var conn client.Client
	var err error

	switch conf := config.(type) {
	case client.HTTPConfig:
		conn, err = client.NewHTTPClient(conf)
		if err != nil {
			return nil, errors.Wrap(err, "error creating HTTPClient")
		}

		_, _, err = conn.Ping(conf.Timeout)
		if err != nil {
			return nil, fmt.Errorf("cannot ping influxdb server: %s", conf.Addr)
		}

		if err := dbCheck(conn, batch.Database); err != nil {
			return nil, errors.Wrapf(err, "check for database %s failed", batch.Database)
		}
	case client.UDPConfig:
		conn, err = client.NewUDPClient(conf)
		if err != nil {
			return nil, errors.Wrap(err, "error creating UDPClient")
		}
	}

	pts := make(chan *client.Point, queueSize)

	bp, err := client.NewBatchPoints(batch)
	if err != nil {
		return nil, errors.Wrap(err, "batchpoints error")
	}

	go func() {
		delay := time.Duration(flush) * time.Second
		tick := time.Tick(delay)
		count := 0
		for {
			select {
			case p := <-pts:
				bp.AddPoint(p)
				count++
				if count < batchSize {
					continue
				}
			case <-tick:
				if len(bp.Points()) == 0 {
					continue
				}
			}
			for {
				if err := conn.Write(bp); err != nil {
					if errFunc != nil {
						errFunc(err)
					}
					time.Sleep(retry)
					continue
				}
				bp, _ = client.NewBatchPoints(batch)
				count = 0
				break
			}
		}
	}()

	return func(key string, tags map[string]string, fields map[string]interface{}, ts time.Time) error {
		pt, err := client.NewPoint(key, tags, fields, ts)
		if err != nil {
			return err
		}
		pts <- pt
		return nil
	}, nil
}

func Crawl(searchUrl string, depth int, fetcher Fetcher, redisClient *redis.Client, influxClient influx.Client) {
	throttle <- 1

	if depth <= 0 {
		return
	}

	fmt.Printf("Depth: %d Crawling: %s\n", depth, searchUrl)

	bp, _ := influx.NewBatchPoints(influx.BatchPointsConfig{
		Database:  "crawler",
		Precision: "s",
	})

	host, err := url.Parse(searchUrl)

	// Send this to our redis queue for indexing
	if err != nil {
		redisClient.LPush("unknown_url_crawler_queue", searchUrl)
	} else {
		redisClient.LPush(host.Host+"_crawler_queue", searchUrl)
	}

	urlcache.lock.Lock()
	urlcache.m[searchUrl] = crawlError
	urlcache.lock.Unlock()

	// let's determine how long it is taking to fetch all urls on a page
	startFetch := time.Now()
	urls, err := fetcher.Fetch(searchUrl)
	crawlTime := time.Since(startFetch)

	if err != nil {
		fmt.Printf("Error fetching results from %s: %s\n", searchUrl, err.Error())

	} else {
		fmt.Printf("Finished crawling %s in %.2f seconds\n", searchUrl, crawlTime.Seconds())
	}

	tags := map[string]string{
		"domain": host.String(),
	}

	fields := map[string]interface{}{
		"urls_found":         len(urls),
		"crawl_time":         crawlTime.Nanoseconds(),
		"total_urls_crawled": len(urlcache.m),
		"urls_by_page":       len(urls),
	}

	point, _ := influx.NewPoint(
		"crawl_usage",
		tags,
		fields,
		time.Now(),
	)

	// add data point to influx
	bp.AddPoint(point)

	if err := influxClient.Write(bp); err != nil {
		log.Printf("Unable to write batch point to influxdb: %s\n", err.Error())
	}

	var wg sync.WaitGroup

	for _, u := range urls {
		if !urlTest.MatchString(u) {
			u = "http://" + host.Host + u
		}

		urlcache.lock.Lock()
		_, crawled := urlcache.m[u]
		urlcache.lock.Unlock()

		if validURL.MatchString(u) && urlTest.MatchString(u) && !crawled {
			wg.Add(1)
			go func(u string, depth int, fetcher Fetcher, redisClient *redis.Client, influxClient influx.Client) {
				defer wg.Done()
				Crawl(u, depth-1, fetcher, redisClient, influxClient)
			}(u, depth, fetcher, redisClient, influxClient)
		}
	}

	<-throttle
	wg.Wait()
}

func watch(client stdinflux.Client, bp stdinflux.BatchPoints, reportTicker <-chan time.Time) {
	for range reportTicker {
		client.Write(bp)
	}
}