Golang Reader.ReadAll方法代碼示例

//Read read csv for handle
func ReadLines(file string, isGbk bool) (lines [][]string, err error) {
	//catch panic
	defer func() {
		if rerr := recover(); rerr != nil {
			err = errors.New(fmt.Sprintf("read csv file: %v, error: %v", file, rerr))
		}
	}()

	//open file
	fi, err := os.Open(file)
	if err != nil {
		return nil, err
	}
	defer fi.Close()
	//get reader
	var reader *csv.Reader
	if !isGbk {
		reader = csv.NewReader(fi)
	} else {
		//transform gbk to utf8
		r := transform.NewReader(fi, simplifiedchinese.GBK.NewDecoder())
		reader = csv.NewReader(r)
	}
	lines, err = reader.ReadAll()
	return
}

// Get is to get OTC csv data.
func (o *OTCLists) Get(category string) ([][]string, error) {
	var (
		csvArrayContent []string
		csvReader       *csv.Reader
		data            []byte
		err             error
		rawData         [][]string
		url             string
	)

	url = fmt.Sprintf("%s%s", utils.OTCHOST, fmt.Sprintf(utils.OTCLISTCSV, fmt.Sprintf("%d/%02d/%02d", o.Date.Year()-1911, o.Date.Month(), o.Date.Day()), category))

	if data, err = hCache.Get(url, false); err == nil {
		csvArrayContent = strings.Split(string(data), "\n")
		if len(csvArrayContent) > 5 {
			csvReader = csv.NewReader(strings.NewReader(strings.Join(csvArrayContent[4:len(csvArrayContent)-1], "\n")))
			if rawData, err = csvReader.ReadAll(); err == nil {
				o.categoryRawData[category] = rawData
				o.formatData(category)
				return rawData, nil
			}
		}
	}
	return nil, err
}

// Get return csv data in array.
func (d *Data) Get() ([][]string, error) {
	if len(d.UnixMapData[d.Date.Unix()]) == 0 {
		data, err := hCache.Get(d.URL(), true)
		if err != nil {
			return nil, fmt.Errorf(errorNetworkFail.Error(), err)
		}
		csvArrayContent := strings.Split(string(data), "\n")
		for i := range csvArrayContent {
			csvArrayContent[i] = strings.TrimSpace(csvArrayContent[i])
		}
		var csvReader *csv.Reader
		if (d.exchange == "tse" && len(csvArrayContent) > 2) || (d.exchange == "otc" && len(csvArrayContent) > 5) {
			if d.exchange == "tse" {
				if d.Name == "" {
					d.Name = strings.Split(csvArrayContent[0], " ")[2]
				}
				csvReader = csv.NewReader(strings.NewReader(strings.Join(csvArrayContent[2:], "\n")))
			} else if d.exchange == "otc" {
				if d.Name == "" {
					d.Name = strings.Split(csvArrayContent[2], ":")[1]
				}
				csvReader = csv.NewReader(strings.NewReader(strings.Join(csvArrayContent[5:len(csvArrayContent)-1], "\n")))
			}
			allData, err := csvReader.ReadAll()
			d.RawData = append(allData, d.RawData...)
			d.UnixMapData[d.Date.Unix()] = allData
			d.clearCache()
			return allData, err
		}
		return nil, errorNotEnoughData
	}
	return d.UnixMapData[d.Date.Unix()], nil
}

func (mfr *MicrophoneFileReader) LoadMicrophones(reader *csv.Reader) (bool, error) {
	records, err := reader.ReadAll()

	if err != nil {
		fmt.Println(err)
		return false, err
	}

	for i := 0; i < len(records); i++ {
		price, err := strconv.ParseFloat(records[i][3], 64)

		if err != nil {
			return false, errors.New("Not able to parse price to float")
		}

		mic := Microphone{
			name:        records[i][0],
			brand:       records[i][1],
			description: records[i][2],
			price:       price,
			url:         records[i][4],
			micType:     records[i][5],
			micStyle:    records[i][6],
		}
		mfr.microphoneList = append(mfr.microphoneList, mic)
	}

	return true, nil
}

func readAllRecord(reader *csv.Reader) {

	recs, err := reader.ReadAll()
	if err != nil {
		log.Fatal(err)
	}

	for _, row := range recs {
		printRow(row)
	}

}

// ReadAll reads all the lines from a csv file and creates a list of objects from that file.
func readAll(file string, c csvReader) ([]csvReader, error) {
	var r *csv.Reader
	var err error

	if len(c.yearS()) == 0 { // no limits on db load
		r, err = newReader(file)
		if err != nil {
			return nil, err
		}
		r.Read() //dispose of the first line if not grepping file
	} else { // limited db load
		/*years := []int{}
		for i := c.start(); i < c.end()+1; i++ {
			years = append(years, i)
		}*/
		r, err = grepYear(file, c.yearS())
		if err != nil {
			return nil, err
		}
	}

	lines, err := r.ReadAll()
	if err != nil {
		return nil, err
	}

	results := make([]csvReader, len(lines))
	for i, l := range lines {
		b, err := c.csvRead(l)
		if err != nil {
			return nil, err
		}
		results[i] = b
	}

	return results, err
}

func executeDiff(cmd *cobra.Command, args []string) {

	var (
		err error

		match  *regexp.Regexp
		ignore *regexp.Regexp

		csvMap  = make(map[string][]string)
		fisList = make([]FileInfos, 0)
		q       = make(chan info)
		wg      = new(sync.WaitGroup)
	)

	if len(args) == 0 {
		cmd.Help()
		return
	}

	// Get glob file args.
	args, err = core.GetGlobArgs(args)
	if err != nil {
		log.Fatalln(err)
	}

	// Recheck args.
	if len(args) <= 1 {
		cmd.Help()
		return
	}

	// Load csv and store.
	for _, csvPath := range args {
		fmt.Println("Open:", csvPath)
		c, err := os.Open(csvPath)
		if err != nil {
			log.Fatalln(err)
		}
		defer c.Close()
		var reader *csv.Reader
		if sjisIn {
			reader = csv.NewReader(transform.NewReader(c, japanese.ShiftJIS.NewDecoder()))
		} else {
			reader = csv.NewReader(c)
		}
		reader.Comma = '\t'
		// Skip header.
		_, err = reader.Read()
		if err != nil {
			log.Fatalln(err)
		}
		left, err := reader.ReadAll()
		if err != nil {
			log.Fatalln(err)
		}

		// Change data to FileInfos struct.
		fis := make(FileInfos, 0)
		for _, r := range left {
			fis = append(fis, *csvToFileInfo(r))
		}
		fisList = append(fisList, fis)
	}

	// Compile if given matches and ignores.
	if len(matches) != 0 {
		match, err = core.CompileStrs(matches)
		if err != nil {
			log.Fatalln(err)
		}
	}
	if len(ignores) != 0 {
		ignore, err = core.CompileStrs(ignores)
		if err != nil {
			log.Fatalln(err)
		}
	}

	for i, one := range fisList {
		wg.Add(1)
		go func(i int, one FileInfos) {
			defer wg.Done()

			// Diff fileinfo.
			for _, oneFi := range one {
				if fileOnly && oneFi.Type == DIR {
					continue
				}
				if dirOnly && oneFi.Type == FILE {
					continue
				}

				// Ignore check.
				if ignore != nil && ignore.MatchString(oneFi.Full) {
					continue
				}

				// Match check.
				if match != nil && !match.MatchString(oneFi.Full) {
					continue
//.........這裏部分代碼省略.........

func write(csvfile string, outfile string, isUtf8 bool) error {
	//open file
	fi, err := os.Open(csvfile)
	if err != nil {
		return err
	}
	defer fi.Close()
	//get reader
	var reader *csv.Reader
	if isUtf8 {
		reader = csv.NewReader(fi)
	} else {
		//transform gbk to utf8
		r := transform.NewReader(fi, simplifiedchinese.GBK.NewDecoder())
		reader = csv.NewReader(r)
	}

	lines, err := reader.ReadAll()
	if err != nil {
		return errors.New(fmt.Sprintf("read error: %v", err))
	}
	lineNum := len(lines)
	if lineNum < 3 {
		return errors.New(fmt.Sprintf("Csv %v is invalid"))
	}
	names, fields, kinds := lines[0], lines[1], lines[2]
	fieldNum := len(names)
	filename := filename(csvfile)

	if outfile == "" {
		outfile = strings.Replace(csvfile, ".csv", ".go", -1)
	}
	//packname
	packname := ""
	outAbs, _ := filepath.Abs(outfile)
	packname = filepath.Base(filepath.Dir(outAbs))
	if packname == "" {
		csvAbs, _ := filepath.Abs(csvfile)
		packname = filepath.Base(filepath.Dir(csvAbs))
	}

	code := fmt.Sprintf("// Code generated by github.com/foolin/gocsv.\n// source: %v\n// DO NOT EDIT! \n\npackage %v\n\ntype %v struct {\n", filepath.Base(csvfile), packname, upper(filename))
	for j := 0; j < fieldNum; j++ {
		name := names[j]
		field := fields[j]
		kind := kinds[j]
		if kind == "float" {
			kind = "float32"
		}
		code = code + fmt.Sprintf("\t%v %v `csv:\"%v\"` //%v\n", upper(field), kind, field, name)
	}
	code = code + "}\n"

	//mkdir
	err = os.MkdirAll(filepath.Dir(outAbs), 0755)
	if err != nil {
		return err
	}

	//write file
	err = ioutil.WriteFile(outfile, []byte(code), 0755)
	if err != nil {
		return err
	}
	log.Printf("write file: %v", outfile)
	return nil
}

func ImportDictionaries() map[string][]*models.SuggestItem {
	var itemMap = make(map[string][]*models.SuggestItem)

	fileInfo, err := ioutil.ReadDir(DataDirectory)
	if err != nil {
		log.Print(err)
		return itemMap
	}
	numberOfDictionaries := 0
	for _, file := range fileInfo {
		if !file.IsDir() && (strings.HasSuffix(file.Name(), ".txt") || strings.HasSuffix(file.Name(), ".txt.gz")) {
			dictionaryFile := fmt.Sprintf("%s%s%s", DataDirectory, string(os.PathSeparator), file.Name())
			dictionaryName := strings.TrimSuffix(strings.TrimSuffix(file.Name(), ".gz"), ".txt")
			log.Printf("Importing dictionary %s from file %s", dictionaryName, dictionaryFile)

			csvFile, err := os.Open(dictionaryFile)
			if err != nil {
				log.Print(err)
				continue
			}
			defer csvFile.Close()
			var csvReader *csv.Reader
			if strings.HasSuffix(file.Name(), ".txt.gz") {
				gzipReader, gzerr := gzip.NewReader(csvFile)
				if gzerr == nil {
					defer gzipReader.Close()
					csvReader = csv.NewReader(gzipReader)
				} else {
					log.Print(gzerr)
					continue
				}
			} else {
				csvReader = csv.NewReader(csvFile)
			}

			csvReader.FieldsPerRecord = 2
			csvReader.Comma = '|'
			csvReader.LazyQuotes = true
			csvReader.TrimLeadingSpace = true

			rawCSVdata, err := csvReader.ReadAll()
			if err != nil {
				log.Print(err)
				continue
			}

			for _, each := range rawCSVdata {
				var suggestItem = new(models.SuggestItem)
				suggestItem.Term = each[0]
				weight, err := strconv.Atoi(each[1])
				if err == nil {
					suggestItem.Weight = weight
					itemMap[dictionaryName] = append(itemMap[dictionaryName], suggestItem)
				}

			}
			numberOfDictionaries++
		}
	}

	log.Printf("Imported %d dictionaries", numberOfDictionaries)
	return itemMap
}