Golang File.WriteAt方法代码示例

func (fst fileStorageTorrent) WriteAt(p []byte, off int64) (n int, err error) {
	for _, fi := range fst.fts.info.UpvertedFiles() {
		if off >= fi.Length {
			off -= fi.Length
			continue
		}
		n1 := len(p)
		if int64(n1) > fi.Length-off {
			n1 = int(fi.Length - off)
		}
		name := fst.fts.fileInfoName(fi)
		os.MkdirAll(filepath.Dir(name), 0770)
		var f *os.File
		f, err = os.OpenFile(name, os.O_WRONLY|os.O_CREATE, 0660)
		if err != nil {
			return
		}
		n1, err = f.WriteAt(p[:n1], off)
		f.Close()
		if err != nil {
			return
		}
		n += n1
		off = 0
		p = p[n1:]
		if len(p) == 0 {
			break
		}
	}
	return
}

func download_segments(threadnum uint, url string, backlog chan Download, instructions chan Download,
	updates chan ProgressUpdate, file *os.File, progressFile *os.File) {

	buf := make([]byte, 8192)
	total := int64(0)
	errorCount := int64(0)

	var down Download
	for {
		select {
		case down = <-backlog:
		case down = <-instructions:
		}
		if down.size == 0 {
			break
		}

		req, err := http.NewRequest("GET", url, nil)
		if err != nil {
			fmt.Fprintln(os.Stderr, "Failed to create request")
			os.Exit(1)
		}

		req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", down.position, down.position+down.size-1))

		resp, err := client.Do(req)
		if err != nil || (resp.StatusCode != 206 && resp.StatusCode != 200) {
			backlog <- down
			updates <- ProgressUpdate{threadnum: threadnum, downloaded: -errorCount}
			time.Sleep(time.Duration(1<<uint64(errorCount)) * 10 * time.Millisecond)
			errorCount++
			continue
		}

		errorCount = 0

		read := int64(0)

		for read < down.size {
			n, err := resp.Body.Read(buf[:])
			if n == 0 && err != nil {
				break
			}

			file.WriteAt(buf[:n], down.position+read)
			read += int64(n)
			total += int64(n)
			updates <- ProgressUpdate{threadnum: threadnum, downloaded: total}
		}

		if read >= down.size {
			buf[0] = finishedSegment
			progressFile.WriteAt(buf[:1], baseOffset+int64(down.segmentNum))
		} else {
			down.position += read
			down.size -= read
			backlog <- down
		}
	}
}

func download(partDownloadInfosChan <-chan PartDownloadInfo, downloadFile *os.File) {
	for partDownloadInfo := range partDownloadInfosChan {

		client := &http.Client{}
		req, err := http.NewRequest("GET", partDownloadInfo.Url, nil)
		if err != nil {
			log.Fatal(err)
		}
		range_header := "bytes=" + strconv.Itoa(partDownloadInfo.OffsetMin) + "-" + strconv.Itoa(partDownloadInfo.OffsetMax-1)
		req.Header.Add("Range", range_header)
		log.Printf("正在下载第%d块大小为%.3fMB文件,请求头为 %s \n", partDownloadInfo.Index, float64(partDownloadInfo.OffsetMax-partDownloadInfo.OffsetMin)/MB, range_header)
		resp, _ := client.Do(req)
		defer resp.Body.Close()

		log.Println("=========>> ", resp.ContentLength, resp.StatusCode, resp.Proto)
		actual_part_size := partDownloadInfo.OffsetMax - partDownloadInfo.OffsetMin
		if resp.ContentLength == int64(actual_part_size) {
			reader, _ := ioutil.ReadAll(resp.Body)
			downloadFile.WriteAt(reader, int64(partDownloadInfo.OffsetMin))
		} else {
			buf := bytes.NewBuffer(make([]byte, actual_part_size))
			n, err := buf.ReadFrom(resp.Body)
			if err != nil && err != io.EOF {
				log.Fatal(err)
			}

			log.Printf("已经读取了%n个字节\n", n)
			downloadFile.WriteAt(buf.Bytes(), int64(partDownloadInfo.OffsetMin))
		}

		wg.Done()
	}
}

func appendWriteFile(fp *os.File, buf []byte) (int, error) {
	stat, err := fp.Stat()
	if err != nil {
		return -1, err
	}

	return fp.WriteAt(buf, stat.Size())
}

func writeMarshaledData(f *os.File, v interface{}) error {
	data, err := json.Marshal(&v)
	if err != nil {
		return err
	}
	_, err = f.WriteAt(data, 0)
	return err
}

// Downloads the given range. In case of an error, sleeps for 10s and tries again.
func DownloadRange(file *File, fp *os.File, offset int64, size int64, rangeWg *sync.WaitGroup, chunkIndex bitField, reportCh chan Report) {
	defer rangeWg.Done()
	reportCh <- Report{ToDownload: size}
	newOffset := offset
	lastByte := offset + size           // The byte we won't be getting
	lastIndex := lastByte/ChunkSize - 1 // The last index we'll fill

	// Creating a custom request because it will have Range header in it
	req, _ := http.NewRequest("GET", file.DownloadUrl(), nil)

	rangeHeader := fmt.Sprintf("bytes=%d-%d", offset, lastByte-1)
	req.Header.Add("Range", rangeHeader)

	// http.DefaultClient does not copy headers while following redirects
	client := &http.Client{
		CheckRedirect: func(req *http.Request, via []*http.Request) error {
			req.Header.Add("Range", rangeHeader)
			return nil
		},
	}

	resp, err := client.Do(req)
	if err != nil {
		log.Println(err)
		return
	}
	defer resp.Body.Close()

	buffer := make([]byte, ChunkSize)
	for {
		nr, er := io.ReadFull(resp.Body, buffer)
		if nr > 0 {
			nw, ew := fp.WriteAt(buffer[0:nr], newOffset)
			nWritten := int64(nw)
			newOffset += nWritten
			currentIndex := newOffset/ChunkSize - 1
			if currentIndex == lastIndex && newOffset != lastByte {
				// dont mark the last bit done without finishing the whole range
			} else {
				chunkIndex.Set(currentIndex)
				fp.WriteAt(chunkIndex, file.Size)
			}
			reportCh <- Report{Downloaded: nWritten}
			if ew != nil {
				log.Println(ew)
				return
			}
		}
		if er == io.EOF || er == io.ErrUnexpectedEOF {
			return
		}
		if er != nil {
			log.Println(er)
			return
		}
	}
}

func (b *Block) writeNode(node *Node, file *os.File, offset uint64,
	buffer *bytes.Buffer, bufferSize uint32, compressedSize uint32) uint64 {

	if b.used && b.nodeID != node.id {
		logger.Fatal("Block Is Used")
	}

	if buffer == nil {
		logger.Fatal("buffer is blank")
	}

	if compressedSize < 1 {
		logger.Fatal("compressedSize < 1")
	}

	if !b.available(compressedSize) {
		logger.Fatal("block unavailable")
	}

	if b.offset > 0 {
		offset = b.offset
	}

	// a, _ := ioutil.ReadFile(file.Name())

	n, err := file.WriteAt(buffer.Bytes(), int64(offset))

	file.Sync()

	if err != nil {
		logger.Fatal(err)
	}

	if n != int(compressedSize) {
		logger.Fatal("Write File Error")
	}

	if b.blockSize == 0 {
		b.blockSize = compressedSize
	}

	if b.blockSize < MinOneBlockDiskSize {
		b.blockSize = MinOneBlockDiskSize
	}

	crcValue := checksum(buffer)

	b.nodeSize = bufferSize
	b.compressedSize = compressedSize
	b.nodeID = node.id
	b.offset = offset
	b.crcValue = crcValue
	b.used = true

	return offset + uint64(b.blockSize)
}

func (file *File) RangeWrite(dest *os.File, start int64, end int64, chMulDow chan int64, partNum int64) {
	var written int64
	var p float32
	var flag = map[int]interface{}{}
	ioReader, err := file.ReqHttpRange(start, end-1)
	reqRangeSize := end - start
	file.WsRespData.Status = "keep"
	if err != nil {
		return
	}
	buf := make([]byte, 32*1024)
	for {
		nr, er := ioReader.Read(buf)
		if nr > 0 {
			nw, ew := dest.WriteAt(buf[0:nr], start)
			start = int64(nw) + start
			if nw > 0 {
				written += int64(nw)
			}
			if ew != nil {
				err = ew
			}
			if nr != nw {
				err = errors.New("short write")
			}

			p = float32(written) / float32(reqRangeSize) * 100
			pp := int(p)
			if pp >= 20 && pp%20 == 0 {
				if flag[pp] != true {
					file.WsRespData.Progress = pp / int(MulSectionDowCount)
					file.WsRespData.PartNum = int(partNum)
					websocket.JSON.Send(file.Ws, file.WsRespData)
					fmt.Printf("%s part%d progress: %v%%\n", file.Name, partNum, int(p))
				}
				flag[pp] = true
			}
		}
		if er != nil {
			if er.Error() == "EOF" {
				//Successfully finish downloading
				if reqRangeSize == written {
					fmt.Printf("%s part%d written  %d\n", file.Name, partNum, written)
					chMulDow <- written
				} else {
					fmt.Printf("%s part%d written  %d (unfinished)\n", file.Name, partNum, written)
					chMulDow <- -1
				}
				break
			}
			fmt.Printf("part%d downloading error : %s\n", partNum, er.Error())
			chMulDow <- -1
			break
		}
	}
}

// WriteAt write from unaligned data buffer via direct I/O
// Use AllocateAligned to avoid extra data fuffer copy
func WriteAt(file *os.File, data []byte, offset int64) (int, error) {
	if alignmentShift(data) == 0 {
		return file.WriteAt(data, offset)
	}
	// Write unaligned
	buf := AllocateAligned(len(data))
	copy(buf, data)
	n, err := file.WriteAt(buf, offset)
	return n, err
}

func (c Client) Get(
	location *os.File,
	contentURL string,
	progressWriter io.Writer,
) error {
	req, err := http.NewRequest("HEAD", contentURL, nil)
	if err != nil {
		return fmt.Errorf("failed to construct HEAD request: %s", err)
	}

	resp, err := c.httpClient.Do(req)
	if err != nil {
		return fmt.Errorf("failed to make HEAD request: %s", err)
	}

	contentURL = resp.Request.URL.String()

	ranges, err := c.ranger.BuildRange(resp.ContentLength)
	if err != nil {
		return fmt.Errorf("failed to construct range: %s", err)
	}

	c.bar.SetOutput(progressWriter)
	c.bar.SetTotal(resp.ContentLength)
	c.bar.Kickoff()

	defer c.bar.Finish()

	var g errgroup.Group
	for _, r := range ranges {
		byteRange := r
		g.Go(func() error {
			respBytes, err := c.retryableRequest(contentURL, byteRange.HTTPHeader)
			if err != nil {
				return fmt.Errorf("failed during retryable request: %s", err)
			}

			bytesWritten, err := location.WriteAt(respBytes, byteRange.Lower)
			if err != nil {
				return fmt.Errorf("failed to write file: %s", err)
			}

			c.bar.Add(bytesWritten)

			return nil
		})
	}

	if err := g.Wait(); err != nil {
		return err
	}

	return nil
}

func (h *regionFileHeader) SetOffset(chunkLoc ChunkXz, offset chunkOffset, file *os.File) os.Error {
	index := indexForChunkLoc(chunkLoc)
	h[index] = offset

	// Write that part of the index.
	var offsetBytes [4]byte
	binary.BigEndian.PutUint32(offsetBytes[:], uint32(offset))
	_, err := file.WriteAt(offsetBytes[:], int64(index)*4)

	return err
}

func (m *fileMonitor) copyRemoteBlocks(cc <-chan content, outFile *os.File, writeWg *sync.WaitGroup) {
	defer writeWg.Done()

	for content := range cc {
		_, err := outFile.WriteAt(content.data, content.offset)
		buffers.Put(content.data)
		if err != nil {
			m.writeError = err
			return
		}
	}
}

func (r *HdCache) WriteAt(p []byte, off int64) []chunk {
	boxI := int(off / int64(r.pieceSize))
	boxOff := int(off % int64(r.pieceSize))

	for i := 0; i < len(p); {
		var box *os.File
		var err error
		if !r.boxExists.IsSet(boxI) { //box doesn't exist, so we'll make one.
			box, err = os.Create(r.boxPrefix + strconv.Itoa(boxI))
			if err != nil {
				log.Panicln("Couldn't create cache file:", err)
				return nil
			}
			r.boxExists.Set(boxI)
			box.Truncate(int64(r.pieceSize))
			r.actualUsage++
		} else { //box exists, so we'll open it
			box, err = os.OpenFile(r.boxPrefix+strconv.Itoa(boxI), os.O_WRONLY, 0777)
			if err != nil {
				log.Println("Error opening cache item we thought we had:", r.boxPrefix+strconv.Itoa(boxI), "error:", err)
				r.removeBox(boxI)
				continue //loop around without incrementing 'i', forget this ever happened
			}
		}
		end := r.pieceSize - boxOff
		if len(p) < end {
			end = len(p)
		}
		copied, err := box.WriteAt(p[i:end], int64(boxOff))
		if err != nil {
			log.Panicln("Error at write cache box:", box.Name(), "error:", err)
		}
		i += copied
		box.Close()
		r.atimes[boxI] = time.Now()
		if copied == r.pieceSize {
			r.isBoxFull.Set(boxI)
		} else {
			if r.isByteSet[boxI].n == 0 {
				r.isByteSet[boxI] = *NewBitset(r.pieceSize)
			}
			for j := boxOff; j < boxOff+copied; j++ {
				r.isByteSet[boxI].Set(j)
			}
		}
		boxI++
		boxOff = 0
	}
	if r.actualUsage > r.getCapacity() {
		return r.trim()
	}
	return nil
}

// Creates a new chunk - extends file
func createChunk(h *HANDLE, chunkIdx int64, buf []byte, off int, sz int) (int, error) {
	var chFile *os.File
	var err error
	var wrote int

	cacheName := h.f.cacheName + "." + strconv.FormatInt(chunkIdx, 10)
	log.WithFields(log.Fields{
		"CacheName": cacheName,
		"ChunkIdx":  chunkIdx,
		"Offset":    off,
		"Size":      sz,
	}).Debug("Revelo::createChunk")

	err = os.MkdirAll(path.Dir(cacheName), 0700) //XXX revisit permission
	if err != nil {
		log.WithFields(log.Fields{
			"CacheName": cacheName,
			"Perm":      0700,
			"Error":     err,
		}).Error("Revelo::createChunk: Cannot MkdirAll")
		return 0, err
	}

	chFile, err = os.OpenFile(cacheName, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600)
	if err != nil {
		// Chunk exists - we should not be here
		log.WithFields(log.Fields{
			"CacheName": cacheName,
			"Size":      sz,
			"ChunkIdx":  chunkIdx,
			"Error":     err,
		}).Error("Revelo::createChunk: Open failed")
		return 0, err
	}
	defer chFile.Close()

	wrote, err = chFile.WriteAt(buf, int64(off))
	if err != nil {
		log.WithFields(log.Fields{
			"ChunkName": cacheName,
			"Off":       off,
			"Size":      sz,
			"Wrote":     wrote,
			"Error":     err,
		}).Error("Revelo::createChunk: WriteAt failed")
		return 0, err
	}

	return wrote, nil
}

func shiftBytesBack(file *os.File, start, offset int64) error {
	stat, err := file.Stat()
	if err != nil {
		return err
	}
	end := stat.Size()

	wrBuf := make([]byte, offset)
	rdBuf := make([]byte, offset)

	wrOffset := offset
	rdOffset := start

	rn, err := file.ReadAt(wrBuf, rdOffset)
	if err != nil && err != io.EOF {
		panic(err)
	}
	rdOffset += int64(rn)

	for {
		if rdOffset >= end {
			break
		}

		n, err := file.ReadAt(rdBuf, rdOffset)
		if err != nil && err != io.EOF {
			return err
		}

		if rdOffset+int64(n) > end {
			n = int(end - rdOffset)
		}

		if _, err := file.WriteAt(wrBuf[:rn], wrOffset); err != nil {
			return err
		}

		rdOffset += int64(n)
		wrOffset += int64(rn)
		copy(wrBuf, rdBuf)
		rn = n
	}

	if _, err := file.WriteAt(wrBuf[:rn], wrOffset); err != nil {
		return err
	}

	return nil
}