Golang seelog.Tracef函数代码示例

func InitClient(conn *net.TCPConn, devid string) *Client {
	client := &Client{
		devId:           devid,
		ctrl:            make(chan bool),
		MsgOut:          make(chan *Pack, 100),
		WaitingChannels: make(map[uint32]chan *Message),
		NextSeqId:       1,
		LastAlive:       time.Now(),
	}
	DevMap.Set(devid, client)

	go func() {
		log.Tracef("start send routine for %s", conn.RemoteAddr().String())
		for {
			select {
			case pack := <-client.MsgOut:
				seqid := pack.client.NextSeqId
				pack.msg.Header.Seq = seqid
				b, _ := pack.msg.Header.Serialize()
				conn.Write(b)
				conn.Write(pack.msg.Data)
				log.Infof("send msg ok, (%s)", string(pack.msg.Data))
				pack.client.NextSeqId += 1
				// add reply channel
				if pack.reply != nil {
					pack.client.WaitingChannels[seqid] = pack.reply
				}
			case <-client.ctrl:
				log.Tracef("leave send routine for %s", conn.RemoteAddr().String())
				return
			}
		}
	}()
	return client
}

func (tz *BadXMLTokenizer) Tokens() <-chan *Token {

	token_channel := make(chan *Token)
	log.Debugf("Created channel %v as part of Tokens(), with"+
		" Scanner = %v", token_channel, tz)

	go func(ret chan *Token, tz *BadXMLTokenizer) {
		for {
			log.Tracef("Scanner calling Next()")
			tok, err := tz.Next()
			log.Tracef("scanner.Next() returned %s, %v", tok, err)
			switch err {
			case nil:
				log.Debugf("Pushing %s into token channel %v",
					tok, ret)
				ret <- tok
			case io.EOF:
				log.Debugf("received EOF, closing channel")
				close(ret)
				log.Debugf("Closed.")
				log.Flush()
				return
				panic("I should have exited the goroutine but " +
					"didn't")
			}
		}
	}(token_channel, tz)

	return token_channel
}

func (s *SyncPair) concurrentSyncS3ToDir(s3url s3Url, bucket *s3.Bucket, targetFiles, sourceFiles map[string]string) error {
	doneChan := newDoneChan(s.Concurrent)
	pool := newPool(s.Concurrent)
	var wg sync.WaitGroup

	for file, _ := range sourceFiles {
		if targetFiles[file] != sourceFiles[file] {
			filePath := strings.Join([]string{s.Target, file}, "/")
			if filepath.Dir(filePath) != "." {
				err := os.MkdirAll(filepath.Dir(filePath), 0755)
				if err != nil {
					return err
				}
			}

			// Get transfer reservation from pool
			log.Tracef("Requesting reservation for '%s'.", filePath)
			<-pool
			log.Tracef("Retrieved reservation for '%s'.", filePath)

			log.Infof("Starting sync: s3://%s/%s -> %s.", bucket.Name, file, filePath)
			wg.Add(1)
			go func(doneChan chan error, filePath string, bucket *s3.Bucket, file string) {
				defer wg.Done()
				writeS3FileToPathRoutine(doneChan, filePath, bucket, file)
				pool <- 1
			}(doneChan, filePath, bucket, file)
		}
	}

	wg.Wait()
	return nil
}

// Test locks:
func Cruft(config *types.Config) {

	log.Infof("[dnsUpdater] Started")

	// Run forever:
	for {

		// Lock the host-list (so we don't change it while another goroutine is using it):
		log.Tracef("[dnsUpdater] Trying to lock config.HostInventoryMutex ...")
		config.HostInventoryMutex.Lock()
		log.Tracef("[dnsUpdater] Locked config.HostInventoryMutex")

		// Show the host-inventory:
		log.Debugf("[dnsUpdater] HostIventory: %v", config.HostInventory)

		// Sleep until the next run:
		log.Tracef("[dnsUpdater] Sleeping for %vs ...", config.DNSUpdateFrequency)
		time.Sleep(time.Duration(config.DNSUpdateFrequency) * time.Second)

		log.Tracef("[dnsUpdater] Unlocking config.HostInventoryMutex ...")
		config.HostInventoryMutex.Unlock()

		time.Sleep(time.Duration(1) * time.Second)

	}

}

// Test locks:
func Cruft(config *types.Config) {

	log.Infof("[hostInventoryUpdater] Started")

	// Run forever:
	for {

		// Lock the host-list (so we don't change it while another goroutine is using it):
		log.Tracef("[hostInventoryUpdater] Trying to lock config.HostInventoryMutex ...")
		config.HostInventoryMutex.Lock()
		log.Tracef("[hostInventoryUpdater] Locked config.HostInventoryMutex")

		// Write some data:
		log.Debugf("[hostInventoryUpdater] Writing 'cruft' to the host-inventory ...")
		config.HostInventory = types.HostInventory{
			Environments: make(map[string]types.Environment),
		}
		config.HostInventory.Environments["cruft"] = types.Environment{}

		// Sleep until the next run:
		log.Tracef("[hostInventoryUpdater] Sleeping for %vs ...", config.HostUpdateFrequency)
		time.Sleep(time.Duration(config.HostUpdateFrequency) * time.Second)

		log.Tracef("[hostInventoryUpdater] Unlocking config.HostInventoryMutex ...")
		config.HostInventoryMutex.Unlock()

		time.Sleep(time.Duration(1) * time.Second)

	}

}

func (s *SyncPair) concurrentSyncDirToS3(s3url s3Url, bucket *s3.Bucket, targetFiles, sourceFiles map[string]string) error {
	doneChan := newDoneChan(s.Concurrent)
	pool := newPool(s.Concurrent)
	var wg sync.WaitGroup

	for file, _ := range sourceFiles {
		// ensure the file has no leading slashes to it compares correctly
		relativeTargetFile := strings.TrimLeft(strings.Join([]string{s3url.Path(), file}, "/"), "/")

		if targetFiles[relativeTargetFile] != sourceFiles[file] {
			filePath := strings.Join([]string{s.Source, file}, "/")
			keyPath := strings.Join([]string{s3url.Key(), file}, "/")

			// Get transfer reservation from pool
			log.Tracef("Requesting reservation for '%s'.", keyPath)
			<-pool
			log.Tracef("Retrieved reservation for '%s'.", keyPath)

			log.Infof("Starting sync: %s -> s3://%s/%s", filePath, bucket.Name, file)
			wg.Add(1)
			go func(doneChan chan error, filePath string, bucket *s3.Bucket, keyPath string) {
				defer wg.Done()
				writeLocalFileToS3Routine(doneChan, filePath, bucket, keyPath)
				pool <- 1
			}(doneChan, filePath, bucket, keyPath)
		}
	}

	// Wait for all routines to finish
	wg.Wait()
	return nil
}

// find all tokens in the channel until it's closed
func (t *Taggers) find(queue chan *string, mysql chan *db.Mysql) {
	i := 0
	for token := range queue {
		if len(*token) == 0 {
			log.Tracef("Caught empty str")
			continue
		}
		conn := <-mysql
		log.Tracef("Searching for token %s", *token)
		kind := t.search_all_tables(token, conn)
		go func() { mysql <- conn }()

		switch kind {
		case -1:
			t.MissingTokens = append(t.MissingTokens, *token)
		case 1:
			t.NamesTokens = append(t.NamesTokens, *token)
		case 2:
			t.DictTokens = append(t.DictTokens, *token)
		case 3:
			t.GeoTokens = append(t.GeoTokens, *token)
		}

		i++
	}

	log.Debugf("Worker, out.  Processed %d tokens", i)
	t.complete <- i
}

func (c *Conn) writeLoop() {
	for {
		select {
		case <-c.exitChan:
			clog.Info("breaking out of writeLoop")
			// Indicate drainReady because we will not pull any more off msgResponseChan
			close(c.drainReady)
			goto exit
		case cmd := <-c.cmdChan:
			err := c.WriteCommand(cmd)
			if err != nil {
				clog.Errorf("error sending command %s - %s", cmd, err)
				c.close()
				continue
			}
		case resp := <-c.msgResponseChan:
			// Decrement this here so it is correct even if we can't respond to nsqd
			msgsInFlight := atomic.AddInt64(&c.messagesInFlight, -1)

			if resp.success {
				clog.Tracef("FIN %s", resp.msg.ID)
				c.delegate.OnMessageFinished(c, resp.msg)
				if resp.backoff {
					c.delegate.OnResume(c)
				}
			} else {
				clog.Tracef("REQ %s", resp.msg.ID)
				c.delegate.OnMessageRequeued(c, resp.msg)
				if resp.backoff {
					c.delegate.OnBackoff(c)
				}
			}

			err := c.WriteCommand(resp.cmd)
			if err != nil {
				clog.Errorf("error sending command %s - %s", resp.cmd, err)
				c.close()
				continue
			}

			if msgsInFlight == 0 &&
				atomic.LoadInt32(&c.closeFlag) == 1 {
				c.close()
				continue
			}
		}
	}

exit:
	c.wg.Done()
	clog.Info("writeLoop exiting")
}

func main() {
	defer seelog.Flush()

	seelog.LoggerFromConfigAsString("formatid=\"debug\"")

	flag.Parse()

	cfg := FtpCfg{*host, *user, *pw, *port}
	fClient, err := NewFtpClient(cfg)
	if err != nil {
		panic(err)
	}
	iClient, err := NewInfluxClient(*surl, *db)
	if err != nil {
		panic(err)
	}

	files := make([]*FtpToInflux, 0)

	scanner := bufio.NewScanner(os.Stdin)
	for scanner.Scan() {
		line := scanner.Text()
		seelog.Tracef("Handle line '%s'", line)
		if strings.HasPrefix(line, commentPrefix) {
			//Comment
			continue
		}
		splittedLine := strings.Split(line, space)
		if len(splittedLine) != 2 {
			seelog.Warnf("Line '%s' has not exactly one space", line)
			continue
		}
		data := &FtpToInflux{splittedLine[0], strings.Split(splittedLine[1], sep)}
		files = append(files, data)
	}

	for _, f := range files {
		seelog.Tracef("Start with file '%s'!", f.Filename)
		buf, err := fClient.Download(f.Filename)
		if err != nil {
			seelog.Warnf("Error downloading file '%s': %v", f.Filename, err)
			continue
		}
		datas := Transform(buf)
		err = iClient.Write(datas, f.Measurements)
		if err != nil {
			seelog.Warnf("Error writing Data: %v", err)
			continue
		}
		seelog.Tracef("File '%s' downloaded and written to %d measurements!", f.Filename, len(f.Measurements))
	}
}

func (r *Consumer) redistributeRDY() {
	if r.inBackoffBlock() {
		return
	}

	numConns := int32(len(r.conns()))
	maxInFlight := r.getMaxInFlight()
	if numConns > maxInFlight {
		clog.Tracef("redistributing RDY state (%d conns > %d max_in_flight)",
			numConns, maxInFlight)
		atomic.StoreInt32(&r.needRDYRedistributed, 1)
	}

	if r.inBackoff() && numConns > 1 {
		clog.Tracef("redistributing RDY state (in backoff and %d conns > 1)", numConns)
		atomic.StoreInt32(&r.needRDYRedistributed, 1)
	}

	if !atomic.CompareAndSwapInt32(&r.needRDYRedistributed, 1, 0) {
		return
	}

	conns := r.conns()
	possibleConns := make([]*Conn, 0, len(conns))
	for _, c := range conns {
		lastMsgDuration := time.Now().Sub(c.LastMessageTime())
		rdyCount := c.RDY()
		clog.Tracef("(%s) rdy: %d (last message received %s)",
			c.String(), rdyCount, lastMsgDuration)
		if rdyCount > 0 && lastMsgDuration > r.config.LowRdyIdleTimeout {
			clog.Tracef("(%s) idle connection, giving up RDY", c.String())
			r.updateRDY(c, 0)
		}
		possibleConns = append(possibleConns, c)
	}

	availableMaxInFlight := int64(maxInFlight) - atomic.LoadInt64(&r.totalRdyCount)
	if r.inBackoff() {
		availableMaxInFlight = 1 - atomic.LoadInt64(&r.totalRdyCount)
	}

	for len(possibleConns) > 0 && availableMaxInFlight > 0 {
		availableMaxInFlight--
		i := r.rng.Int() % len(possibleConns)
		c := possibleConns[i]
		// delete
		possibleConns = append(possibleConns[:i], possibleConns[i+1:]...)
		clog.Tracef("(%s) redistributing RDY", c.String())
		r.updateRDY(c, 1)
	}
}

func parseXML(sc *scanner.Scanner) (*Token, bool) {

	var entity = new(bytes.Buffer)
	token := new(Token)

	// Skip the '<'
	sc.Scan()

	switch sc.Peek() {
	case '/':
		token.Type = XMLEndToken
		sc.Next()
	case '!':
		log.Tracef("parseXML skipping comment")
		next := sc.Next()
		for next != '>' {
			next = sc.Next()
		}
		return nil, false
	default:
		token.Type = XMLStartToken
	}

	log.Tracef("parseXML creating %s element", token.Type)

	for {
		tok := sc.Scan()
		log.Tracef("parseXML found %s. Token is %v. Entity is: '%s'",
			sc.TokenText(),
			tok,
			entity.String())

		switch {
		case tok == '>':
			token.Text = entity.String()
			return token, true

		case unicode.IsSpace(tok):
			return nil, false

		default:
			log.Tracef("parseXML appending %s to string",
				sc.TokenText())
			entity.WriteString(sc.TokenText())

		}
	}
}

// SendPublication via AMQP
func SendPublication(pub Publication, InstanceID string) error {
	log.Tracef("[Raven] Sending publication to topic: %s", pub.Topic())

	if !Connected {
		return fmt.Errorf("[Raven] Error sending publication, raven not connected")
	}

	err := Publisher.channel.Publish(
		TOPIC_EXCHANGE, // publish to topic exchange
		pub.Topic(),    // routing key = topic
		false,          // mandatory
		false,          // immediate
		amqp.Publishing{
			Headers: amqp.Table{
				"messageType": "publication",
				"topic":       pub.Topic(),
				"sessionID":   pub.SessionID(),
			},
			ContentType:     pub.ContentType(),
			ContentEncoding: contentEncoding,
			Body:            pub.Payload(),
			DeliveryMode:    deliveryMode,
			Priority:        defaultPriority,
			MessageId:       pub.MessageID(),
			ReplyTo:         InstanceID,
			// a bunch of application/implementation-specific fields
		})

	if err != nil {
		return fmt.Errorf("Error sending publication: %s", err)
	}

	return nil
}

// JsonRequest JSON encodes and sends the object in reqData.ReqValue (if any) to the specified URL.
// Optional method arguments are passed using the RequestData object.
// Relevant RequestData fields:
// ReqHeaders: additional HTTP header values to add to the request.
// ExpectedStatus: the allowed HTTP response status values, else an error is returned.
// ReqValue: the data object to send.
// RespValue: the data object to decode the result into.
func (c *Client) JsonRequest(method, url string, reqData *RequestData) (err error) {
	err = nil
	var body []byte
	if reqData.Params != nil {
		url += "?" + reqData.Params.Encode()
	}
	if sbody, ok := reqData.ReqValue.(string); ok {
		body = []byte(sbody)
	} else if reqData.ReqValue != nil {
		body, err = json.Marshal(reqData.ReqValue)
		if err != nil {
			err = errors.Newf(err, "failed marshalling the request body")
			return
		}
	}
	headers := c.createHeaders(reqData.ReqHeaders, contentTypeJSON)
	respBody, statusCode, err := c.sendRequest(
		method, url, bytes.NewReader(body), len(body), headers, reqData.ExpectedStatus)
	reqData.StatusCode = statusCode
	log.Tracef("%s:%s", method, url)

	if err != nil {
		return
	}
	err = unmarshallResponse(respBody, reqData)
	return
}

// This func is designed to be run as a goroutine. It
// listens for messages on a channel and sends them to a peer.
func (p *peer) peerWriter(errorChan chan peerMessage) {
	log.Infof("[%s] Writing messages to peer[%s]", p.taskID, p.address)
	var lastWriteTime time.Time

	for msg := range p.writeChan {
		now := time.Now()
		if len(msg) == 0 {
			// This is a keep-alive message.
			if now.Sub(lastWriteTime) < 2*time.Minute {
				continue
			}
			log.Tracef("[%s] Sending keep alive to peer[%s]", p.taskID, p.address)
		}
		lastWriteTime = now

		//log.Debugf("[%s] Sending message to peer[%s], length=%v", p.taskID, p.address, uint32(len(msg)))
		err := writeNBOUint32(p.flowctrlWriter, uint32(len(msg)))
		if err != nil {
			log.Error(err)
			break
		}
		_, err = p.flowctrlWriter.Write(msg)
		if err != nil {
			log.Errorf("[%s] Failed to write a message to peer[%s], length=%v, err=%v", p.taskID, p.address, len(msg), err)
			break
		}
	}

	log.Infof("[%s] Exiting Writing messages to peer[%s]", p.taskID, p.address)
	errorChan <- peerMessage{p, nil}
}

// Authorise tests auth
func (a *simpleAuthoriser) Authorise(req *Request) errors.Error {

	// If we require neither a role or a user, then there is no need to authorise
	if !a.requireUser && !a.requireRole {
		log.Debugf("Skipping auth from %s to %s, as neither user or role required", req.From(), req.Destination())
		return nil
	}

	// Otherwise, authorise this request
	scope := req.Auth()
	log.Tracef("Scope user: %v", scope.AuthUser())
	if a.requireUser && !scope.IsAuth() {
		return errors.Forbidden("com.hailocab.kernel.auth.notsignedin", fmt.Sprintf("Must be signed in to call this endpoint[endpoint=%s, service=%s, from=%s]",
			req.Endpoint(), req.Service(), req.From()), "201")
	}
	if a.requireRole {
		matchesRole := false
		for _, r := range a.roles {
			if scope.HasAccess(r) {
				matchesRole = true
				break
			}
		}
		if !matchesRole {
			if scope.HasTriedAuth() {
				return errors.Forbidden("com.hailocab.kernel.auth.badrole", fmt.Sprintf("Must be signed in to call this endpoint[endpoint=%s, service=%s, from=%s]",
					req.Endpoint(), req.Service(), req.From()), "201")
			}
			// Instrument when service to service auth fails
			inst.Counter(1.0, "auth.servicetoservice.failed", 1)
			return BadRoleError(req)
		}
	}
	return nil
}