Golang conf.Config类代码示例

func NewReaper(cnf *conf.Config) *Reaper {
	rpr := &Reaper{
		lg:           common.NewLogger("reaper", cnf),
		spanExpiryMs: cnf.GetInt64(conf.HTRACE_SPAN_EXPIRY_MS),
		heartbeats:   make(chan interface{}, 1),
	}
	if rpr.spanExpiryMs >= MAX_SPAN_EXPIRY_MS {
		rpr.spanExpiryMs = MAX_SPAN_EXPIRY_MS
	} else if rpr.spanExpiryMs <= 0 {
		rpr.spanExpiryMs = MAX_SPAN_EXPIRY_MS
	}
	rpr.hb = NewHeartbeater("ReaperHeartbeater",
		cnf.GetInt64(conf.HTRACE_REAPER_HEARTBEAT_PERIOD_MS), rpr.lg)
	rpr.exited.Add(1)
	go rpr.run()
	rpr.hb.AddHeartbeatTarget(&HeartbeatTarget{
		name:       "reaper",
		targetChan: rpr.heartbeats,
	})
	var when string
	if rpr.spanExpiryMs >= MAX_SPAN_EXPIRY_MS {
		when = "never"
	} else {
		when = "after " + time.Duration(rpr.spanExpiryMs).String()
	}
	rpr.lg.Infof("Initializing span reaper: span time out = %s.\n", when)
	return rpr
}

func CreateRestServer(cnf *conf.Config, store *dataStore) (*RestServer, error) {
	var err error
	rsv := &RestServer{}
	rsv.listener, err = net.Listen("tcp", cnf.Get(conf.HTRACE_WEB_ADDRESS))
	if err != nil {
		return nil, err
	}
	var success bool
	defer func() {
		if !success {
			rsv.Close()
		}
	}()
	rsv.lg = common.NewLogger("rest", cnf)

	r := mux.NewRouter().StrictSlash(false)

	r.Handle("/server/info", &serverInfoHandler{lg: rsv.lg}).Methods("GET")

	serverStatsH := &serverStatsHandler{dataStoreHandler: dataStoreHandler{
		store: store, lg: rsv.lg}}
	r.Handle("/server/stats", serverStatsH).Methods("GET")

	writeSpansH := &writeSpansHandler{dataStoreHandler: dataStoreHandler{
		store: store, lg: rsv.lg}}
	r.Handle("/writeSpans", writeSpansH).Methods("POST")

	queryH := &queryHandler{lg: rsv.lg, dataStoreHandler: dataStoreHandler{store: store}}
	r.Handle("/query", queryH).Methods("GET")

	span := r.PathPrefix("/span").Subrouter()
	findSidH := &findSidHandler{dataStoreHandler: dataStoreHandler{store: store, lg: rsv.lg}}
	span.Handle("/{id}", findSidH).Methods("GET")

	findChildrenH := &findChildrenHandler{dataStoreHandler: dataStoreHandler{store: store,
		lg: rsv.lg}}
	span.Handle("/{id}/children", findChildrenH).Methods("GET")

	// Default Handler. This will serve requests for static requests.
	webdir := os.Getenv("HTRACED_WEB_DIR")
	if webdir == "" {
		webdir, err = filepath.Abs(filepath.Join(filepath.Dir(os.Args[0]), "..", "web"))

		if err != nil {
			return nil, err
		}
	}

	rsv.lg.Infof("Serving static files from %s\n.", webdir)
	r.PathPrefix("/").Handler(http.FileServer(http.Dir(webdir))).Methods("GET")

	// Log an error message for unknown non-GET requests.
	r.PathPrefix("/").Handler(&logErrorHandler{lg: rsv.lg})

	go http.Serve(rsv.listener, r)

	rsv.lg.Infof("Started REST server on %s...\n", rsv.listener.Addr().String())
	success = true
	return rsv, nil
}

func NewMetricsSink(cnf *conf.Config) *MetricsSink {
	return &MetricsSink{
		lg:               common.NewLogger("metrics", cnf),
		maxMtx:           cnf.GetInt(conf.HTRACE_METRICS_MAX_ADDR_ENTRIES),
		HostSpanMetrics:  make(common.SpanMetricsMap),
		wsLatencyCircBuf: NewCircBufU32(LATENCY_CIRC_BUF_SIZE),
	}
}

// A golang client for htraced.
// TODO: fancier APIs for streaming spans in the background, optimize TCP stuff
func NewClient(cnf *conf.Config, testHooks *TestHooks) (*Client, error) {
	hcl := Client{testHooks: testHooks}
	hcl.restAddr = cnf.Get(conf.HTRACE_WEB_ADDRESS)
	if testHooks != nil && testHooks.HrpcDisabled {
		hcl.hrpcAddr = ""
	} else {
		hcl.hrpcAddr = cnf.Get(conf.HTRACE_HRPC_ADDRESS)
	}
	return &hcl, nil
}

func CreateHrpcServer(cnf *conf.Config, store *dataStore) (*HrpcServer, error) {
	lg := common.NewLogger("hrpc", cnf)
	hsv := &HrpcServer{
		Server: rpc.NewServer(),
		hand: &HrpcHandler{
			lg:    lg,
			store: store,
		},
	}
	var err error
	hsv.listener, err = net.Listen("tcp", cnf.Get(conf.HTRACE_HRPC_ADDRESS))
	if err != nil {
		return nil, err
	}
	hsv.Server.Register(hsv.hand)
	go hsv.run()
	lg.Infof("Started HRPC server on %s...\n", hsv.listener.Addr().String())
	return hsv, nil
}

func CreateDataStore(cnf *conf.Config, writtenSpans chan *common.Span) (*dataStore, error) {
	// Get the configuration.
	clearStored := cnf.GetBool(conf.HTRACE_DATA_STORE_CLEAR)
	dirsStr := cnf.Get(conf.HTRACE_DATA_STORE_DIRECTORIES)
	dirs := strings.Split(dirsStr, conf.PATH_LIST_SEP)

	var err error
	lg := common.NewLogger("datastore", cnf)
	store := &dataStore{lg: lg, shards: []*shard{}, WrittenSpans: writtenSpans}

	// If we return an error, close the store.
	defer func() {
		if err != nil {
			store.Close()
			store = nil
		}
	}()

	store.readOpts = levigo.NewReadOptions()
	store.readOpts.SetFillCache(true)
	store.writeOpts = levigo.NewWriteOptions()
	store.writeOpts.SetSync(false)

	// Open all shards
	for idx := range dirs {
		path := dirs[idx] + conf.PATH_SEP + "db"
		var shd *shard
		shd, err = CreateShard(store, cnf, path, clearStored)
		if err != nil {
			lg.Errorf("Error creating shard %s: %s\n", path, err.Error())
			return nil, err
		}
		store.shards = append(store.shards, shd)
	}
	for idx := range store.shards {
		shd := store.shards[idx]
		shd.exited = make(chan bool, 1)
		go shd.processIncoming()
	}
	return store, nil
}

func CreateDataStore(cnf *conf.Config, writtenSpans *common.Semaphore) (*dataStore, error) {
	dld := NewDataStoreLoader(cnf)
	defer dld.Close()
	err := dld.Load()
	if err != nil {
		dld.lg.Errorf("Error loading datastore: %s\n", err.Error())
		return nil, err
	}
	store := &dataStore{
		lg:           dld.lg,
		shards:       make([]*shard, len(dld.shards)),
		readOpts:     dld.readOpts,
		writeOpts:    dld.writeOpts,
		WrittenSpans: writtenSpans,
		msink:        NewMetricsSink(cnf),
		hb: NewHeartbeater("DatastoreHeartbeater",
			cnf.GetInt64(conf.HTRACE_DATASTORE_HEARTBEAT_PERIOD_MS), dld.lg),
		rpr:     NewReaper(cnf),
		startMs: common.TimeToUnixMs(time.Now().UTC()),
	}
	spanBufferSize := cnf.GetInt(conf.HTRACE_DATA_STORE_SPAN_BUFFER_SIZE)
	for shdIdx := range store.shards {
		shd := &shard{
			store:      store,
			ldb:        dld.shards[shdIdx].ldb,
			path:       dld.shards[shdIdx].path,
			incoming:   make(chan []*IncomingSpan, spanBufferSize),
			heartbeats: make(chan interface{}, 1),
		}
		shd.exited.Add(1)
		go shd.processIncoming()
		store.shards[shdIdx] = shd
		store.hb.AddHeartbeatTarget(&HeartbeatTarget{
			name:       fmt.Sprintf("shard(%s)", shd.path),
			targetChan: shd.heartbeats,
		})
	}
	dld.DisownResources()
	return store, nil
}

// Create a new datastore loader.
// Initializes the loader, but does not load any leveldb instances.
func NewDataStoreLoader(cnf *conf.Config) *DataStoreLoader {
	dld := &DataStoreLoader{
		lg:          common.NewLogger("datastore", cnf),
		ClearStored: cnf.GetBool(conf.HTRACE_DATA_STORE_CLEAR),
	}
	dld.readOpts = levigo.NewReadOptions()
	dld.readOpts.SetFillCache(true)
	dld.readOpts.SetVerifyChecksums(false)
	dld.writeOpts = levigo.NewWriteOptions()
	dld.writeOpts.SetSync(false)
	dirsStr := cnf.Get(conf.HTRACE_DATA_STORE_DIRECTORIES)
	rdirs := strings.Split(dirsStr, conf.PATH_LIST_SEP)
	// Filter out empty entries
	dirs := make([]string, 0, len(rdirs))
	for i := range rdirs {
		if strings.TrimSpace(rdirs[i]) != "" {
			dirs = append(dirs, rdirs[i])
		}
	}
	dld.shards = make([]*ShardLoader, len(dirs))
	for i := range dirs {
		dld.shards[i] = &ShardLoader{
			dld:  dld,
			path: dirs[i] + conf.PATH_SEP + "db",
		}
	}
	dld.openOpts = levigo.NewOptions()
	cacheSize := cnf.GetInt(conf.HTRACE_LEVELDB_CACHE_SIZE)
	dld.openOpts.SetCache(levigo.NewLRUCache(cacheSize))
	dld.openOpts.SetParanoidChecks(false)
	writeBufferSize := cnf.GetInt(conf.HTRACE_LEVELDB_WRITE_BUFFER_SIZE)
	if writeBufferSize > 0 {
		dld.openOpts.SetWriteBufferSize(writeBufferSize)
	}
	maxFdPerShard := dld.calculateMaxOpenFilesPerShard()
	if maxFdPerShard > 0 {
		dld.openOpts.SetMaxOpenFiles(maxFdPerShard)
	}
	return dld
}

func CreateHrpcServer(cnf *conf.Config, store *dataStore,
	testHooks *hrpcTestHooks) (*HrpcServer, error) {
	lg := common.NewLogger("hrpc", cnf)
	numHandlers := cnf.GetInt(conf.HTRACE_NUM_HRPC_HANDLERS)
	if numHandlers < 1 {
		lg.Warnf("%s must be positive: using 1 handler.\n", conf.HTRACE_NUM_HRPC_HANDLERS)
		numHandlers = 1
	}
	if numHandlers > MAX_HRPC_HANDLERS {
		lg.Warnf("%s cannot be more than %d: using %d handlers\n",
			conf.HTRACE_NUM_HRPC_HANDLERS, MAX_HRPC_HANDLERS, MAX_HRPC_HANDLERS)
		numHandlers = MAX_HRPC_HANDLERS
	}
	hsv := &HrpcServer{
		Server: rpc.NewServer(),
		hand: &HrpcHandler{
			lg:    lg,
			store: store,
		},
		cdcs:     make(chan *HrpcServerCodec, numHandlers),
		shutdown: make(chan interface{}),
		ioTimeo: time.Millisecond *
			time.Duration(cnf.GetInt64(conf.HTRACE_HRPC_IO_TIMEOUT_MS)),
		testHooks: testHooks,
	}
	for i := 0; i < numHandlers; i++ {
		hsv.cdcs <- &HrpcServerCodec{
			lg:  lg,
			hsv: hsv,
			msgpackHandle: codec.MsgpackHandle{
				WriteExt: true,
			},
		}
	}
	var err error
	hsv.listener, err = net.Listen("tcp", cnf.Get(conf.HTRACE_HRPC_ADDRESS))
	if err != nil {
		return nil, err
	}
	hsv.Server.Register(hsv.hand)
	hsv.exited.Add(1)
	go hsv.run()
	lg.Infof("Started HRPC server on %s with %d handler routines. "+
		"ioTimeo=%s.\n", hsv.listener.Addr().String(), numHandlers,
		hsv.ioTimeo.String())
	return hsv, nil
}

func NewClient(cnf *conf.Config) (*Client, error) {
	hcl := Client{}
	hcl.restAddr = cnf.Get(conf.HTRACE_WEB_ADDRESS)
	hcl.hrpcAddr = cnf.Get(conf.HTRACE_HRPC_ADDRESS)
	return &hcl, nil
}

func parseConf(faculty string, cnf *conf.Config) (string, Level) {
	facultyLogPathKey := faculty + "." + conf.HTRACE_LOG_PATH
	var facultyLogPath string
	if cnf.Contains(facultyLogPathKey) {
		facultyLogPath = cnf.Get(facultyLogPathKey)
	} else {
		facultyLogPath = cnf.Get(conf.HTRACE_LOG_PATH)
	}
	facultyLogLevelKey := faculty + "." + conf.HTRACE_LOG_LEVEL
	var facultyLogLevelStr string
	if cnf.Contains(facultyLogLevelKey) {
		facultyLogLevelStr = cnf.Get(facultyLogLevelKey)
	} else {
		facultyLogLevelStr = cnf.Get(conf.HTRACE_LOG_LEVEL)
	}
	level, err := LevelFromString(facultyLogLevelStr)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error configuring log level: %s.  Using TRACE.\n")
		level = TRACE
	}
	return facultyLogPath, level
}

func CreateShard(store *dataStore, cnf *conf.Config, path string,
	clearStored bool) (*shard, error) {
	lg := store.lg
	if clearStored {
		fi, err := os.Stat(path)
		if err != nil && !os.IsNotExist(err) {
			lg.Errorf("Failed to stat %s: %s\n", path, err.Error())
			return nil, err
		}
		if fi != nil {
			err = os.RemoveAll(path)
			if err != nil {
				lg.Errorf("Failed to clear existing datastore directory %s: %s\n",
					path, err.Error())
				return nil, err
			}
			lg.Infof("Cleared existing datastore directory %s\n", path)
		}
	}
	err := os.MkdirAll(path, 0777)
	if err != nil {
		lg.Errorf("Failed to MkdirAll(%s): %s\n", path, err.Error())
		return nil, err
	}
	var shd *shard
	openOpts := levigo.NewOptions()
	defer openOpts.Close()
	newlyCreated := false
	ldb, err := levigo.Open(path, openOpts)
	if err == nil {
		store.lg.Infof("LevelDB opened %s\n", path)
	} else {
		store.lg.Debugf("LevelDB failed to open %s: %s\n", path, err.Error())
		openOpts.SetCreateIfMissing(true)
		ldb, err = levigo.Open(path, openOpts)
		if err != nil {
			store.lg.Errorf("LevelDB failed to create %s: %s\n", path, err.Error())
			return nil, err
		}
		store.lg.Infof("Created new LevelDB instance in %s\n", path)
		newlyCreated = true
	}
	defer func() {
		if shd == nil {
			ldb.Close()
		}
	}()
	lv, err := readLayoutVersion(store, ldb)
	if err != nil {
		store.lg.Errorf("Got error while reading datastore version for %s: %s\n",
			path, err.Error())
		return nil, err
	}
	if newlyCreated && (lv == UNKNOWN_LAYOUT_VERSION) {
		err = writeDataStoreVersion(store, ldb, CURRENT_LAYOUT_VERSION)
		if err != nil {
			store.lg.Errorf("Got error while writing datastore version for %s: %s\n",
				path, err.Error())
			return nil, err
		}
		store.lg.Tracef("Wrote layout version %d to shard at %s.\n",
			CURRENT_LAYOUT_VERSION, path)
	} else if lv != CURRENT_LAYOUT_VERSION {
		versionName := "unknown"
		if lv != UNKNOWN_LAYOUT_VERSION {
			versionName = fmt.Sprintf("%d", lv)
		}
		store.lg.Errorf("Can't read old datastore.  Its layout version is %s, but this "+
			"software is at layout version %d.  Please set %s to clear the datastore "+
			"on startup, or clear it manually.\n", versionName,
			CURRENT_LAYOUT_VERSION, conf.HTRACE_DATA_STORE_CLEAR)
		return nil, errors.New(fmt.Sprintf("Invalid layout version: got %s, expected %d.",
			versionName, CURRENT_LAYOUT_VERSION))
	} else {
		store.lg.Tracef("Found layout version %d in %s.\n", lv, path)
	}
	spanBufferSize := cnf.GetInt(conf.HTRACE_DATA_STORE_SPAN_BUFFER_SIZE)
	shd = &shard{store: store, ldb: ldb, path: path,
		incoming: make(chan *common.Span, spanBufferSize)}
	return shd, nil
}