Golang patricia.NewTrie函数代码示例

// NewTransientFilesystem returns a new TransientFilesystem.  Expects an
// absolute path to the location on disk.  Returns an error if the path cannot
// be created, or if a file already exists in that location.
// If create is true, then the directory will be created if it does not yet exist.
func NewTransientFilesystem(absPath string, create bool) (*TransientFilesystem, error) {
	if !filepath.IsAbs(absPath) {
		return nil, ErrPathNotAbsolute
	}
	fi, err := os.Stat(absPath)
	// short circuit for create
	if create && os.IsNotExist(err) {
		if err = os.MkdirAll(absPath, 0755); err != nil {
			return nil, err
		}
		fs := &TransientFilesystem{trie: trie.NewTrie(), rootPath: absPath}
		if err = fs.Sync(); err != nil {
			return nil, err
		}
		return fs, nil
	}
	if err != nil {
		return nil, fmt.Errorf("Unable to stat path %s", absPath)
	}
	if !fi.IsDir() {
		return nil, fmt.Errorf("Specified path %s is not a directory", absPath)
	}
	fs := &TransientFilesystem{trie: trie.NewTrie(), rootPath: absPath}
	if err = fs.Sync(); err != nil {
		return nil, err
	}
	return fs, nil
}

// NewLocationMuxer returns a new LocationMuxer for the given Locations
func NewLocationMuxer(locations []*types.Location) (*LocationMuxer, error) {
	lm := new(LocationMuxer)
	lm.locationTrie = patricia.NewTrie()
	for _, location := range locations {
		switch {
		case isLocationType(location, none):
			lm.locationTrie.Set([]byte(location.Name), location)
		case isLocationType(location, caseInsensitiveRegular):
			if err := lm.addRegexForLocation(fmt.Sprintf(`(?i)%s`, location.Name[len(caseInsensitiveRegular):]), location); err != nil {
				return nil, fmt.Errorf("Location %s gave error while being parsed to regex: %s", location, err)
			}
		case isLocationType(location, caseSensitiveRegular):
			if err := lm.addRegexForLocation(location.Name[len(caseSensitiveRegular):], location); err != nil {
				return nil, fmt.Errorf("Location %s gave error while being parsed to regex: %s", location, err)
			}
		case isLocationType(location, exact):
			lm.locationTrie.Set([]byte(location.Name[len(exact)-1:]), location)
		case isLocationType(location, bestNonRegular):
			lm.locationTrie.Set([]byte(location.Name[len(bestNonRegular)-1:]), location)
		default:
			return nil, fmt.Errorf("Location %s is not parsable", location)

		}
	}

	return lm, nil
}

// ValidateProto returns an error if the given proto has problems
// that would cause InitFromProto to fail.
func ValidateProto(config *tableaclpb.Config) (err error) {
	t := patricia.NewTrie()
	for _, group := range config.TableGroups {
		for _, name := range group.TableNamesOrPrefixes {
			var prefix patricia.Prefix
			if strings.HasSuffix(name, "%") {
				prefix = []byte(strings.TrimSuffix(name, "%"))
			} else {
				prefix = []byte(name + "\000")
			}
			if bytes.Contains(prefix, []byte("%")) {
				return fmt.Errorf("got: %s, '%%' means this entry is a prefix and should not appear in the middle of name or prefix", name)
			}
			overlapVisitor := func(_ patricia.Prefix, item patricia.Item) error {
				return fmt.Errorf("conflicting entries: %q overlaps with %q", name, item)
			}
			if err := t.VisitSubtree(prefix, overlapVisitor); err != nil {
				return err
			}
			if err := t.VisitPrefixes(prefix, overlapVisitor); err != nil {
				return err
			}
			t.Insert(prefix, name)
		}
	}
	return nil
}

func main() {
	key := flag.String("key", "", "key in dot notation")
	version := flag.Bool("v", false, "show version and exit")

	flag.Parse()

	if *version {
		fmt.Println(Version)
		os.Exit(0)
	}

	if *key == "" {
		log.Fatal("key is required")
	}

	var reader *bufio.Reader
	if flag.NArg() < 1 {
		reader = bufio.NewReader(os.Stdin)
	} else {
		file, err := os.Open(flag.Arg(0))
		if err != nil {
			log.Fatal(err)
		}
		defer file.Close()
		reader = bufio.NewReader(file)
	}

	trie := patricia.NewTrie()

	for {
		line, err := reader.ReadString('\n')
		if err == io.EOF {
			break
		}
		if err != nil {
			log.Fatal(err)
		}
		doc := make(map[string]interface{})
		err = json.Unmarshal([]byte(line), &doc)
		if err != nil {
			log.Fatal(err)
		}
		val, err := StringValue(*key, doc)
		if err != nil {
			log.Fatal(err)
		}
		if trie.Match(patricia.Prefix(val)) {
			log.Println("SKIP")
			continue
		}
		b, err := json.Marshal(doc)
		if err != nil {
			log.Fatal(err)
		}
		trie.Insert(patricia.Prefix(val), 1)
		fmt.Println(string(b))
	}
}

// Constructor function for Exchange, nothing special here.
func New() *Exchange {
	mu := new(sync.RWMutex)
	return &Exchange{
		trie:           patricia.NewTrie(),
		topicForHandle: make(map[Handle]Topic),
		mu:             mu,
		cond:           sync.NewCond(mu),
	}
}

func NewTruncIndex(ids []string) (idx *TruncIndex) {
	idx = &TruncIndex{
		ids:  make(map[string]struct{}),
		trie: patricia.NewTrie(),
	}
	for _, id := range ids {
		idx.addId(id)
	}
	return
}

func LoadDictionaries() {
	var newDictionaryMap = make(map[string]*patricia.Trie)
	var itemMap = ImportDictionaries()

	numPrefixes := 0
	numSuggestions := 0
	numDictionaries := 0

	for dictionaryName, suggestItems := range itemMap {
		numDictionaries++
		log.Print("Loading dictionary " + dictionaryName)
		// First see if the trie already exists
		var trie, ok = newDictionaryMap[dictionaryName]
		if !ok {
			trie = patricia.NewTrie()
		}

		// Great, we have a trie, now let's see if prefixes for the
		// suggestItems exist in the trie
		for _, suggestItem := range suggestItems {
			numSuggestions++
			//Tokenize the suggested term by whitespace.  Each token will become a prefix in the trie
			var tokens = strings.Fields(suggestItem.Term)
			tokens = append(tokens, suggestItem.Term)
			for _, token := range tokens {
				numPrefixes++
				//TODO: use ascii folding
				lowerToken := strings.ToLower(token)
				// The values in the trie are sorted sets of SuggestItems
				trieItem := trie.Get([]byte(lowerToken))
				if trieItem != nil {
					suggestItemSet := trieItem.(treeset.Set)
					//If the set already exists, add the new suggestion to the set
					suggestItemSet.Add(suggestItem)

				} else {
					// Otherwise create a new set, add the SuggestItem, and insert it into
					// the trie using the lowercase token for the prefix
					suggestItemSet := treeset.NewWith(models.SuggestItemComparator)
					//					log.Printf("Inserting suggestion item %s (%s)", lowerToken, suggestItem.Term)
					suggestItemSet.Add(suggestItem)
					trie.Insert(patricia.Prefix([]byte(lowerToken)), *suggestItemSet)
				}
			}
		}
		newDictionaryMap[dictionaryName] = trie
		log.Print("Dictionary " + dictionaryName + " loaded")
	}
	//Atomic swap
	DictionaryMap = newDictionaryMap
	log.Printf("All dictionaries updated")
}

// NewTruncIndex creates a new TruncIndex and initializes with a list of IDs.
func NewTruncIndex(ids []string) (idx *TruncIndex) {
	idx = &TruncIndex{
		ids: make(map[string]struct{}),

		// Change patricia max prefix per node length,
		// because our len(ID) always 64
		trie: patricia.NewTrie(patricia.MaxPrefixPerNode(64)),
	}
	for _, id := range ids {
		idx.addID(id)
	}
	return
}

// NewDefaultRouter creates a very basic WAMP router.
func NewNode() Router {
	log.Println("Creating new Rabric Node")

	trie := patricia.NewTrie()
	r := &Realm{URI: "pd"}
	trie.Insert(patricia.Prefix("pd"), r)

	return &node{
		realms:                make(map[URI]Realm),
		sessionOpenCallbacks:  []func(uint, string){},
		sessionCloseCallbacks: []func(uint, string){},
		root: trie,
	}
}

func (_ Test) Patricia() {
	e.InfoLog.Println("\nPatricia:")
	printItem := func(prefix patricia.Prefix, item patricia.Item) error {
		e.InfoLog.Println(string(prefix), item.(uint32))
		return nil
	}
	trie := patricia.NewTrie()

	for i := range ExampleWords {
		trie.Insert(patricia.Prefix(ExampleWords[i]), ExampleUint32[i])
	}
	trie.Set(patricia.Prefix("coca cola"), 188)

	e.InfoLog.Println("SubTree:")
	trie.VisitSubtree(patricia.Prefix("blost"), printItem)
	e.InfoLog.Println("Prefixes:")
	trie.VisitPrefixes(patricia.Prefix("borte med blesten mega"), printItem)

	trie.Delete(patricia.Prefix("coca cola"))
	trie.DeleteSubtree(patricia.Prefix("blost"))

	e.InfoLog.Println("What is left:")
	trie.Visit(printItem)
}

// Sync brings the in-memory struct and underlying file system into sync.
// Walk the entire tree and recreate it from scratch.
func (fs *TransientFilesystem) Sync() error {
	newTrie := trie.NewTrie()
	err := filepath.Walk(fs.RootPath(), func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		fi, err := os.Stat(path)
		if err != nil {
			return err
		}
		if !newTrie.Insert(trie.Prefix(path), fi) {
			return fmt.Errorf("Path %s already exists in tree", path)
		}
		return nil
	})
	if err != nil {
		return err
	}

	fs.lock.Lock()
	defer fs.lock.Unlock()
	fs.trie = newTrie
	return nil
}

func dodiffRecursive(firstClnt, secondClnt client.Client, ch chan DiffMessage) {
	firstTrie := patricia.NewTrie()
	secondTrie := patricia.NewTrie()
	wg := new(sync.WaitGroup)

	type urlAttr struct {
		Size int64
		Type os.FileMode
	}

	wg.Add(1)
	go func(ch chan<- DiffMessage) {
		defer wg.Done()
		for firstContentCh := range firstClnt.List(true) {
			if firstContentCh.Err != nil {
				ch <- DiffMessage{
					Error: firstContentCh.Err.Trace(firstClnt.URL().String()),
				}
				return
			}
			firstTrie.Insert(patricia.Prefix(firstContentCh.Content.Name), urlAttr{firstContentCh.Content.Size, firstContentCh.Content.Type})
		}
	}(ch)
	wg.Add(1)
	go func(ch chan<- DiffMessage) {
		defer wg.Done()
		for secondContentCh := range secondClnt.List(true) {
			if secondContentCh.Err != nil {
				ch <- DiffMessage{
					Error: secondContentCh.Err.Trace(secondClnt.URL().String()),
				}
				return
			}
			secondTrie.Insert(patricia.Prefix(secondContentCh.Content.Name), urlAttr{secondContentCh.Content.Size, secondContentCh.Content.Type})
		}
	}(ch)

	doneCh := make(chan struct{})
	defer close(doneCh)
	go func(doneCh <-chan struct{}) {
		cursorCh := cursorAnimate()
		for {
			select {
			case <-time.Tick(100 * time.Millisecond):
				if !globalQuietFlag && !globalJSONFlag {
					console.PrintC("\r" + "Scanning.. " + string(<-cursorCh))
				}
			case <-doneCh:
				return
			}
		}
	}(doneCh)
	wg.Wait()
	doneCh <- struct{}{}
	if !globalQuietFlag && !globalJSONFlag {
		console.Printf("%c[2K\n", 27)
		console.Printf("%c[A", 27)
	}

	matchNameCh := make(chan string, 10000)
	go func(matchNameCh chan<- string) {
		itemFunc := func(prefix patricia.Prefix, item patricia.Item) error {
			matchNameCh <- string(prefix)
			return nil
		}
		firstTrie.Visit(itemFunc)
		defer close(matchNameCh)
	}(matchNameCh)
	for matchName := range matchNameCh {
		firstURLDelimited := firstClnt.URL().String()[:strings.LastIndex(firstClnt.URL().String(), string(firstClnt.URL().Separator))+1]
		secondURLDelimited := secondClnt.URL().String()[:strings.LastIndex(secondClnt.URL().String(), string(secondClnt.URL().Separator))+1]
		firstURL := firstURLDelimited + matchName
		secondURL := secondURLDelimited + matchName
		if !secondTrie.Match(patricia.Prefix(matchName)) {
			ch <- DiffMessage{
				FirstURL:  firstURL,
				SecondURL: secondURL,
				Diff:      "only-in-first",
			}
		} else {
			firstURLAttr := firstTrie.Get(patricia.Prefix(matchName)).(urlAttr)
			secondURLAttr := secondTrie.Get(patricia.Prefix(matchName)).(urlAttr)

			if firstURLAttr.Type.IsRegular() {
				if !secondURLAttr.Type.IsRegular() {
					ch <- DiffMessage{
						FirstURL:  firstURL,
						SecondURL: secondURL,
						Diff:      "type",
					}
					continue
				}
			}

			if firstURLAttr.Type.IsDir() {
				if !secondURLAttr.Type.IsDir() {
					ch <- DiffMessage{
						FirstURL:  firstURL,
						SecondURL: secondURL,
						Diff:      "type",
//.........这里部分代码省略.........

func initPrefixes() *gotrie.Trie {
	prefixes := gotrie.NewTrie()

	prefixes.Insert(gotrie.Prefix("1403"), "CA")
	prefixes.Insert(gotrie.Prefix("1587"), "CA")
	prefixes.Insert(gotrie.Prefix("1780"), "CA")
	prefixes.Insert(gotrie.Prefix("1825"), "CA")
	prefixes.Insert(gotrie.Prefix("1236"), "CA")
	prefixes.Insert(gotrie.Prefix("1250"), "CA")
	prefixes.Insert(gotrie.Prefix("1604"), "CA")
	prefixes.Insert(gotrie.Prefix("1672"), "CA")
	prefixes.Insert(gotrie.Prefix("1778"), "CA")
	prefixes.Insert(gotrie.Prefix("1204"), "CA")
	prefixes.Insert(gotrie.Prefix("1431"), "CA")
	prefixes.Insert(gotrie.Prefix("1506"), "CA")
	prefixes.Insert(gotrie.Prefix("1709"), "CA")
	prefixes.Insert(gotrie.Prefix("1782"), "CA")
	prefixes.Insert(gotrie.Prefix("1902"), "CA")
	prefixes.Insert(gotrie.Prefix("1548"), "CA")
	prefixes.Insert(gotrie.Prefix("1249"), "CA")
	prefixes.Insert(gotrie.Prefix("1289"), "CA")
	prefixes.Insert(gotrie.Prefix("1343"), "CA")
	prefixes.Insert(gotrie.Prefix("1365"), "CA")
	prefixes.Insert(gotrie.Prefix("1387"), "CA")
	prefixes.Insert(gotrie.Prefix("1416"), "CA")
	prefixes.Insert(gotrie.Prefix("1437"), "CA")
	prefixes.Insert(gotrie.Prefix("1519"), "CA")
	prefixes.Insert(gotrie.Prefix("1613"), "CA")
	prefixes.Insert(gotrie.Prefix("1647"), "CA")
	prefixes.Insert(gotrie.Prefix("1705"), "CA")
	prefixes.Insert(gotrie.Prefix("1742"), "CA")
	prefixes.Insert(gotrie.Prefix("1807"), "CA")
	prefixes.Insert(gotrie.Prefix("1905"), "CA")
	prefixes.Insert(gotrie.Prefix("1782"), "CA")
	prefixes.Insert(gotrie.Prefix("1902"), "CA")
	prefixes.Insert(gotrie.Prefix("1418"), "CA")
	prefixes.Insert(gotrie.Prefix("1438"), "CA")
	prefixes.Insert(gotrie.Prefix("1450"), "CA")
	prefixes.Insert(gotrie.Prefix("1514"), "CA")
	prefixes.Insert(gotrie.Prefix("1579"), "CA")
	prefixes.Insert(gotrie.Prefix("1581"), "CA")
	prefixes.Insert(gotrie.Prefix("1819"), "CA")
	prefixes.Insert(gotrie.Prefix("1873"), "CA")
	prefixes.Insert(gotrie.Prefix("1306"), "CA")
	prefixes.Insert(gotrie.Prefix("1639"), "CA")
	prefixes.Insert(gotrie.Prefix("1867"), "CA")
	prefixes.Insert(gotrie.Prefix("1"), "US")
	prefixes.Insert(gotrie.Prefix("1242"), "BS")
	prefixes.Insert(gotrie.Prefix("1246"), "BB")
	prefixes.Insert(gotrie.Prefix("1264"), "AI")
	prefixes.Insert(gotrie.Prefix("1268"), "AG")
	prefixes.Insert(gotrie.Prefix("1284"), "VG")
	prefixes.Insert(gotrie.Prefix("1340"), "VI")
	prefixes.Insert(gotrie.Prefix("1345"), "KY")
	prefixes.Insert(gotrie.Prefix("1441"), "BM")
	prefixes.Insert(gotrie.Prefix("1473"), "GD")
	prefixes.Insert(gotrie.Prefix("1649"), "TC")
	prefixes.Insert(gotrie.Prefix("1664"), "MS")
	prefixes.Insert(gotrie.Prefix("1670"), "MP")
	prefixes.Insert(gotrie.Prefix("1671"), "GU")
	prefixes.Insert(gotrie.Prefix("1684"), "AS")
	prefixes.Insert(gotrie.Prefix("1721"), "SX")
	prefixes.Insert(gotrie.Prefix("1758"), "LC")
	prefixes.Insert(gotrie.Prefix("1767"), "DM")
	prefixes.Insert(gotrie.Prefix("1784"), "VC")
	prefixes.Insert(gotrie.Prefix("1787"), "PR")
	prefixes.Insert(gotrie.Prefix("1809"), "DO")
	prefixes.Insert(gotrie.Prefix("1829"), "DO")
	prefixes.Insert(gotrie.Prefix("1849"), "DO")
	prefixes.Insert(gotrie.Prefix("1868"), "TT")
	prefixes.Insert(gotrie.Prefix("1869"), "KN")
	prefixes.Insert(gotrie.Prefix("1876"), "JM")
	prefixes.Insert(gotrie.Prefix("1939"), "PR")
	prefixes.Insert(gotrie.Prefix("20"), "EG")
	prefixes.Insert(gotrie.Prefix("211"), "SS")
	prefixes.Insert(gotrie.Prefix("212"), "MA")
	prefixes.Insert(gotrie.Prefix("213"), "DZ")
	prefixes.Insert(gotrie.Prefix("216"), "TN")
	prefixes.Insert(gotrie.Prefix("218"), "LY")
	prefixes.Insert(gotrie.Prefix("220"), "GM")
	prefixes.Insert(gotrie.Prefix("221"), "SN")
	prefixes.Insert(gotrie.Prefix("222"), "MR")
	prefixes.Insert(gotrie.Prefix("223"), "ML")
	prefixes.Insert(gotrie.Prefix("224"), "GN")
	prefixes.Insert(gotrie.Prefix("225"), "CI")
	prefixes.Insert(gotrie.Prefix("226"), "BF")
	prefixes.Insert(gotrie.Prefix("227"), "NE")
	prefixes.Insert(gotrie.Prefix("228"), "TG")
	prefixes.Insert(gotrie.Prefix("229"), "BJ")
	prefixes.Insert(gotrie.Prefix("230"), "MU")
	prefixes.Insert(gotrie.Prefix("231"), "LR")
	prefixes.Insert(gotrie.Prefix("232"), "SL")
	prefixes.Insert(gotrie.Prefix("233"), "GH")
	prefixes.Insert(gotrie.Prefix("234"), "NG")
	prefixes.Insert(gotrie.Prefix("235"), "TD")
	prefixes.Insert(gotrie.Prefix("236"), "CF")
	prefixes.Insert(gotrie.Prefix("237"), "CM")
	prefixes.Insert(gotrie.Prefix("238"), "CV")
	prefixes.Insert(gotrie.Prefix("239"), "ST")
	prefixes.Insert(gotrie.Prefix("240"), "GQ")
//.........这里部分代码省略.........

func deltaSourceTargets(sourceClnt client.Client, targetClnts []client.Client) <-chan mirrorURLs {
	mirrorURLsCh := make(chan mirrorURLs)
	go func() {
		defer close(mirrorURLsCh)
		sourceTrie := patricia.NewTrie()
		targetTries := make([]*patricia.Trie, len(targetClnts))
		wg := new(sync.WaitGroup)

		wg.Add(1)
		go func() {
			defer wg.Done()
			for sourceContentCh := range sourceClnt.List(true) {
				if sourceContentCh.Err != nil {
					mirrorURLsCh <- mirrorURLs{Error: sourceContentCh.Err.Trace()}
					return
				}
				if sourceContentCh.Content.Type.IsRegular() {
					sourceTrie.Insert(patricia.Prefix(sourceContentCh.Content.Name), sourceContentCh.Content.Size)
				}
			}
		}()

		for i, targetClnt := range targetClnts {
			wg.Add(1)
			go func(i int, targetClnt client.Client) {
				defer wg.Done()
				targetTrie := patricia.NewTrie()
				for targetContentCh := range targetClnt.List(true) {
					if targetContentCh.Err != nil {
						mirrorURLsCh <- mirrorURLs{Error: targetContentCh.Err.Trace()}
						return
					}
					if targetContentCh.Content.Type.IsRegular() {
						targetTrie.Insert(patricia.Prefix(targetContentCh.Content.Name), struct{}{})
					}
				}
				targetTries[i] = targetTrie
			}(i, targetClnt)
		}
		wg.Wait()

		matchNameCh := make(chan string, 10000)
		go func(matchNameCh chan<- string) {
			itemFunc := func(prefix patricia.Prefix, item patricia.Item) error {
				matchNameCh <- string(prefix)
				return nil
			}
			sourceTrie.Visit(itemFunc)
			defer close(matchNameCh)
		}(matchNameCh)
		for matchName := range matchNameCh {
			sourceContent := new(client.Content)
			var targetContents []*client.Content
			for i, targetTrie := range targetTries {
				if !targetTrie.Match(patricia.Prefix(matchName)) {
					sourceURLDelimited := sourceClnt.URL().String()[:strings.LastIndex(sourceClnt.URL().String(),
						string(sourceClnt.URL().Separator))+1]
					newTargetURLParse := *targetClnts[i].URL()
					newTargetURLParse.Path = filepath.Join(newTargetURLParse.Path, matchName)
					sourceContent.Size = sourceTrie.Get(patricia.Prefix(matchName)).(int64)
					sourceContent.Name = sourceURLDelimited + matchName
					targetContents = append(targetContents, &client.Content{Name: newTargetURLParse.String()})
				}
			}
			mirrorURLsCh <- mirrorURLs{
				SourceContent:  sourceContent,
				TargetContents: targetContents,
			}
		}
	}()
	return mirrorURLsCh
}