Golang x.Check函数代码示例

func parseDefaultConfig(l string) (uint32, error) {
	// If we have already seen a default config line, and n has a value then we
	// log.Fatal.
	if groupConfig.n != 0 {
		return 0, fmt.Errorf("Default config can only be defined once: %v", l)
	}
	l = strings.TrimSpace(l)
	conf := strings.Split(l, " ")
	// + in (fp % n + k) is optional.
	if !(len(conf) == 5 || len(conf) == 3) || conf[0] != "fp" || conf[1] != "%" {
		return 0, fmt.Errorf("Default config format should be like: %v", "default: fp % n + k")
	}

	var err error
	var n uint64
	n, err = strconv.ParseUint(conf[2], 10, 32)
	x.Check(err)
	groupConfig.n = uint32(n)
	x.AssertTrue(groupConfig.n != 0)
	if len(conf) == 5 {
		if conf[3] != "+" {
			return 0, fmt.Errorf("Default config format should be like: %v", "default: fp % n + k")
		}
		n, err = strconv.ParseUint(conf[4], 10, 32)
		groupConfig.k = uint32(n)
		x.Check(err)
	}
	return groupConfig.k, nil
}

func toRDF(buf *bytes.Buffer, item kv) {
	pl := item.list
	for _, p := range pl.Postings {
		x.Check2(buf.WriteString(item.prefix))

		if p.Uid == math.MaxUint64 && !bytes.Equal(p.Value, nil) {
			// Value posting
			// Convert to appropriate type
			typ := stype.ValueForType(stype.TypeID(p.ValType))
			x.Check(typ.UnmarshalBinary(p.Value))
			str, err := typ.MarshalText()
			x.Check(err)

			x.Check2(buf.WriteString(fmt.Sprintf("\"%s\"", str)))
			if p.ValType == uint32(stype.GeoID) {
				x.Check2(buf.WriteString(fmt.Sprintf("^^<geo:geojson> ")))
			} else if p.ValType != uint32(stype.BytesID) {
				x.Check2(buf.WriteString(fmt.Sprintf("^^<xs:%s> ", typ.Type().Name)))
			}
			x.Check2(buf.WriteString(" .\n"))
			return
		}
		x.Check2(buf.WriteString(fmt.Sprintf("<_uid_:%#x> .\n", p.Uid)))
	}
}

func main() {
	flag.Parse()
	f, err := os.Open(*geoData)
	x.Check(err)

	gr, err := gzip.NewReader(f)
	x.Check(err)

	//var strBuf bytes.Buffer
	//bufReader := bufio.NewReader(gr)
	dec := json.NewDecoder(gr)

	countryToGeo := make(map[string]string)
	findFeatureArray(dec)
	for dec.More() {
		var f geojson.Feature
		err := dec.Decode(&f)
		fmt.Println(f.Properties["NAME_LONG"])
		gg, err := geojson.Marshal(f.Geometry)
		ggg := strings.Replace(string(gg), "\"", "'", -1)
		country, ok := f.Properties["NAME_LONG"].(string)
		if ok {
			countryToGeo[country] = ggg
		}
		//fmt.Printf("\"%s\"", ggg)
		if err != nil {
			fmt.Println(err)
		}
	}
	gr.Close()
	f.Close()

	f, err = os.Open(*rdf)
	x.Check(err)

	gr, err = gzip.NewReader(f)
	x.Check(err)

	scanner := bufio.NewScanner(gr)

	out, err := os.Create("countryGeoData")
	x.Check(err)
	defer out.Close()
	count1, count2 := 0, 0
	for scanner.Scan() {
		line := scanner.Text()
		if strings.Contains(line, "@en") {
			items := strings.Split(line, "\t")
			country := strings.Trim(strings.Split(items[2], "@")[0], "\"")
			fmt.Println(country)
			if geoD, ok := countryToGeo[country]; ok {
				count1++
				out.WriteString(fmt.Sprintf("%s <loc> \"%s\"^^<geo:geojson> .\n", items[0], geoD))
			} else {
				count2++
			}
		}
	}
	fmt.Println(count1, count2)
}

func makeRequest(mutation chan string, c *uint64, wg *sync.WaitGroup) {
	var counter uint64
	for m := range mutation {
		counter = atomic.AddUint64(c, 1)
		if counter%100 == 0 {
			fmt.Printf("Request: %v\n", counter)
		}
	RETRY:
		req, err := http.NewRequest("POST", *dgraph, strings.NewReader(body(m)))
		x.Check(err)
		res, err := hc.Do(req)
		if err != nil {
			fmt.Printf("Retrying req: %d. Error: %v\n", counter, err)
			time.Sleep(5 * time.Millisecond)
			goto RETRY
		}

		body, err := ioutil.ReadAll(res.Body)
		x.Check(err)
		x.Check(json.Unmarshal(body, &r))
		if r.Code != "ErrorOk" {
			log.Fatalf("Error while performing mutation: %v, err: %v", m, r.Message)
		}
	}
	wg.Done()
}

func makeRequests(mutation chan string, wg *sync.WaitGroup) {
	for m := range mutation {
		counter := atomic.AddUint64(&s.mutations, 1)
		if counter%100 == 0 {
			num := atomic.LoadUint64(&s.rdfs)
			dur := time.Since(s.start)
			rate := float64(num) / dur.Seconds()
			fmt.Printf("[Request: %6d] Total RDFs done: %8d RDFs per second: %7.0f\r", counter, num, rate)
		}
	RETRY:
		req, err := http.NewRequest("POST", *dgraph, strings.NewReader(body(m)))
		x.Check(err)
		res, err := hc.Do(req)
		if err != nil {
			fmt.Printf("Retrying req: %d. Error: %v\n", counter, err)
			time.Sleep(5 * time.Millisecond)
			goto RETRY
		}

		body, err := ioutil.ReadAll(res.Body)
		x.Check(err)
		if err = json.Unmarshal(body, &r); err != nil {
			// Not doing x.Checkf(json.Unmarshal..., "Response..", string(body))
			// to ensure that we don't try to convert body from []byte to string
			// when there's no errors.
			x.Checkf(err, "HTTP Status: %s Response body: %s.",
				http.StatusText(res.StatusCode), string(body))
		}
		if r.Code != "ErrorOk" {
			log.Fatalf("Error while performing mutation: %v, err: %v", m, r.Message)
		}
	}
	wg.Done()
}

func convertToEdges(ctx context.Context, nquads []rdf.NQuad) (mutationResult, error) {
	var edges []*task.DirectedEdge
	var mr mutationResult

	newUids := make(map[string]uint64)
	for _, nq := range nquads {
		if strings.HasPrefix(nq.Subject, "_new_:") {
			newUids[nq.Subject] = 0
		} else if !strings.HasPrefix(nq.Subject, "_uid_:") {
			uid, err := rdf.GetUid(nq.Subject)
			x.Check(err)
			newUids[nq.Subject] = uid
		}

		if len(nq.ObjectId) > 0 {
			if strings.HasPrefix(nq.ObjectId, "_new_:") {
				newUids[nq.ObjectId] = 0
			} else if !strings.HasPrefix(nq.ObjectId, "_uid_:") {
				uid, err := rdf.GetUid(nq.ObjectId)
				x.Check(err)
				newUids[nq.ObjectId] = uid
			}
		}
	}

	if len(newUids) > 0 {
		if err := worker.AssignUidsOverNetwork(ctx, newUids); err != nil {
			x.TraceError(ctx, x.Wrapf(err, "Error while GetOrAssignUidsOverNetwork"))
			return mr, err
		}
	}

	for _, nq := range nquads {
		// Get edges from nquad using newUids.
		edge, err := nq.ToEdgeUsing(newUids)
		if err != nil {
			x.TraceError(ctx, x.Wrapf(err, "Error while converting to edge: %v", nq))
			return mr, err
		}
		edges = append(edges, edge)
	}

	resultUids := make(map[string]uint64)
	// Strip out _new_: prefix from the keys.
	for k, v := range newUids {
		if strings.HasPrefix(k, "_new_:") {
			resultUids[k[6:]] = v
		}
	}

	mr = mutationResult{
		edges:   edges,
		newUids: resultUids,
	}
	return mr, nil
}

// ParseConfig parses a group config provided by reader.
func ParseConfig(r io.Reader) error {
	groupConfig = config{}
	scanner := bufio.NewScanner(r)
	// To keep track of last groupId seen across lines. If we the groups ids are
	// not sequential, we log.Fatal.
	var curGroupId uint32
	// If after seeing line with default config, we see other lines, we log.Fatal.
	// Default config should be specified as the last line, so that we can check
	// accurately that k in (fp % N + k) generates consecutive groups.
	seenDefault := false
	for scanner.Scan() {
		l := scanner.Text()
		// Skip empty lines and comments.
		if l == "" || strings.HasPrefix(l, "//") {
			continue
		}
		c := strings.Split(l, ":")
		if len(c) < 2 {
			return fmt.Errorf("Incorrect format for config line: %v", l)
		}
		if c[0] == "default" {
			seenDefault = true
			k, err := parseDefaultConfig(c[1])
			if err != nil {
				return err
			}
			if k == 0 {
				continue
			}
			if k > curGroupId {
				return fmt.Errorf("k in (fp mod N + k) should be <= the last groupno %v.",
					curGroupId)
			}
		} else {
			// There shouldn't be a line after the default config line.
			if seenDefault {
				return fmt.Errorf("Default config should be specified as the last line. Found %v",
					l)
			}
			groupId, err := strconv.ParseUint(c[0], 10, 32)
			x.Check(err)
			if curGroupId != uint32(groupId) {
				return fmt.Errorf("Group ids should be sequential and should start from 0. "+
					"Found %v, should have been %v", groupId, curGroupId)
			}
			curGroupId++
			err = parsePredicates(uint32(groupId), c[1])
			if err != nil {
				return err
			}
		}
	}
	x.Check(scanner.Err())
	return nil
}

func main() {
	flag.Parse()
	logrus.SetLevel(logrus.DebugLevel)
	var srcl, dstl []R
	f, bufReader := getReader(*src)
	var err error

	srcCount := 0
	var strBuf bytes.Buffer
	for {
		err = x.ReadLine(bufReader, &strBuf)
		if err != nil {
			break
		}
		srcCount++
		rnq, err := rdf.Parse(strBuf.String())
		x.Checkf(err, "Unable to parse line: [%v]", strBuf.String())
		srcl = append(srcl, convert(rnq))
	}
	if err != nil && err != io.EOF {
		err := x.Errorf("Error while reading file: %v", err)
		log.Fatalf("%+v", err)
	}
	x.Check(f.Close())
	fmt.Println("Source done")

	f, bufReader = getReader(*dst)
	dstCount := 0
	for {
		err = x.ReadLine(bufReader, &strBuf)
		if err != nil {
			break
		}
		dstCount++
		rnq, err := rdf.Parse(strBuf.String())
		x.Checkf(err, "Unable to parse line: [%v]", strBuf.String())
		dstl = append(dstl, convert(rnq))
	}
	if err != nil && err != io.EOF {
		err := x.Errorf("Error while reading file: %v", err)
		log.Fatalf("%+v", err)
	}
	x.Check(f.Close())

	fmt.Printf("Src: [%d] Dst: [%d]\n", srcCount, dstCount)
	sort.Sort(ByR(srcl))
	sort.Sort(ByR(dstl))
	fmt.Println("Comparing now")
	//for i := 0; i < 100; i++ {
	//fmt.Printf("[S,D] %v %v\n", srcl[i], dstl[i])
	//}
	compare(srcl, dstl)
}

func checkFlagsAndInitDirs() {
	numCpus := *numcpu
	if len(*cpuprofile) > 0 {
		f, err := os.Create(*cpuprofile)
		x.Check(err)
		pprof.StartCPUProfile(f)
	}

	prev := runtime.GOMAXPROCS(numCpus)
	log.Printf("num_cpu: %v. prev_maxprocs: %v. Set max procs to num cpus",
		numCpus, prev)
	// Create parent directories for postings, uids and mutations
	x.Check(os.MkdirAll(*postingDir, 0700))
}

func main() {
	flag.Parse()
	f, err := os.Open(*file)
	x.Check(err)
	defer f.Close()
	gr, err := gzip.NewReader(f)
	x.Check(err)

	hc = http.Client{Timeout: time.Minute}
	mutation := make(chan string, 3*(*concurrent))

	var count uint64 = 0
	var wg sync.WaitGroup
	for i := 0; i < *concurrent; i++ {
		wg.Add(1)
		go makeRequest(mutation, &count, &wg)
	}

	var buf bytes.Buffer
	bufReader := bufio.NewReader(gr)
	num := 0
	var rdfCount uint64 = 0
	for {
		err = readLine(bufReader, &buf)
		if err != nil {
			break
		}
		buf.WriteRune('\n')

		if num >= *numRdf {
			mutation <- buf.String()
			buf.Reset()
			num = 0
		}
		rdfCount++
		num++
	}
	if err != io.EOF {
		log.Fatalf("Error while reading file: %+v", err)
	}
	if buf.Len() > 0 {
		mutation <- buf.String()
	}
	close(mutation)

	wg.Wait()
	fmt.Println("Number of RDF's parsed: ", rdfCount)
	fmt.Println("Number of mutations run: ", count)
}

// stringHelper does simple DFS to convert FilterTree to string.
func (t *FilterTree) stringHelper(buf *bytes.Buffer) {
	x.AssertTrue(t != nil)
	if t.Func != nil && len(t.Func.Name) > 0 {
		// Leaf node.
		_, err := buf.WriteRune('(')
		x.Check(err)
		_, err = buf.WriteString(t.Func.Name)
		x.Check(err)

		if len(t.Func.Attr) > 0 {
			args := make([]string, len(t.Func.Args)+1)
			args[0] = t.Func.Attr
			copy(args[1:], t.Func.Args)

			for _, arg := range args {
				_, err = buf.WriteString(" \"")
				x.Check(err)
				_, err = buf.WriteString(arg)
				x.Check(err)
				_, err := buf.WriteRune('"')
				x.Check(err)
			}
		}
		_, err = buf.WriteRune(')')
		x.Check(err)
		return
	}
	// Non-leaf node.
	_, err := buf.WriteRune('(')
	x.Check(err)
	switch t.Op {
	case "&":
		_, err = buf.WriteString("AND")
	case "|":
		_, err = buf.WriteString("OR")
	default:
		err = x.Errorf("Unknown operator: %q", t.Op)
	}
	x.Check(err)

	for _, c := range t.Child {
		_, err = buf.WriteRune(' ')
		x.Check(err)
		c.stringHelper(buf)
	}
	_, err = buf.WriteRune(')')
	x.Check(err)
}

func (p *poolsi) connect(addr string) {
	if addr == *myAddr {
		return
	}
	p.RLock()
	_, has := p.all[addr]
	p.RUnlock()
	if has {
		return
	}

	pool := newPool(addr, 5)
	query := new(Payload)
	query.Data = make([]byte, 10)
	x.Check2(rand.Read(query.Data))

	conn, err := pool.Get()
	x.Checkf(err, "Unable to connect")

	c := NewWorkerClient(conn)
	resp, err := c.Echo(context.Background(), query)
	x.Checkf(err, "Unable to Echo")
	x.AssertTrue(bytes.Equal(resp.Data, query.Data))
	x.Check(pool.Put(conn))
	fmt.Printf("Connection with %q successful.\n", addr)

	p.Lock()
	defer p.Unlock()
	_, has = p.all[addr]
	if has {
		return
	}
	p.all[addr] = pool
}

func main() {
	rand.Seed(time.Now().UnixNano())
	x.Init()
	checkFlagsAndInitDirs()

	ps, err := store.NewStore(*postingDir)
	x.Checkf(err, "Error initializing postings store")
	defer ps.Close()

	if len(*schemaFile) > 0 {
		err = schema.Parse(*schemaFile)
		x.Checkf(err, "Error while loading schema: %s", *schemaFile)
	}
	// Posting will initialize index which requires schema. Hence, initialize
	// schema before calling posting.Init().
	posting.Init(ps)
	worker.Init(ps)
	x.Check(group.ParseGroupConfig(*conf))

	// Setup external communication.
	che := make(chan error, 1)
	go setupServer(che)
	go worker.StartRaftNodes(*walDir)

	if err := <-che; !strings.Contains(err.Error(),
		"use of closed network connection") {
		log.Fatal(err)
	}
}

// InitAndStartNode gets called after having at least one membership sync with the cluster.
func (n *node) InitAndStartNode(wal *raftwal.Wal) {
	restart, err := n.initFromWal(wal)
	x.Check(err)

	if restart {
		fmt.Printf("RESTARTING\n")
		n.raft = raft.RestartNode(n.cfg)

	} else {
		if groups().HasPeer(n.gid) {
			n.joinPeers()
			n.raft = raft.StartNode(n.cfg, nil)

		} else {
			peers := []raft.Peer{{ID: n.id}}
			n.raft = raft.StartNode(n.cfg, peers)
			// Trigger election, so this node can become the leader of this single-node cluster.
			n.canCampaign = true
		}
	}
	go n.processCommitCh()
	go n.Run()
	// TODO: Find a better way to snapshot, so we don't lose the membership
	// state information, which isn't persisted.
	// go n.snapshotPeriodically()
	go n.batchAndSendMessages()
}

func (w *grpcWorker) RaftMessage(ctx context.Context, query *Payload) (*Payload, error) {
	if ctx.Err() != nil {
		return &Payload{}, ctx.Err()
	}

	for idx := 0; idx < len(query.Data); {
		sz := int(binary.LittleEndian.Uint32(query.Data[idx : idx+4]))
		idx += 4
		msg := raftpb.Message{}
		if idx+sz-1 > len(query.Data) {
			return &Payload{}, x.Errorf(
				"Invalid query. Size specified: %v. Size of array: %v\n", sz, len(query.Data))
		}
		if err := msg.Unmarshal(query.Data[idx : idx+sz]); err != nil {
			x.Check(err)
		}
		if msg.Type != raftpb.MsgHeartbeat && msg.Type != raftpb.MsgHeartbeatResp {
			fmt.Printf("RECEIVED: %v %v-->%v\n", msg.Type, msg.From, msg.To)
		}
		if err := w.applyMessage(ctx, msg); err != nil {
			return &Payload{}, err
		}
		idx += sz
	}
	// fmt.Printf("Got %d messages\n", count)
	return &Payload{}, nil
}