Golang Needle.IsChunkedManifest方法代码示例

func (vs *VolumeServer) tryHandleChunkedFile(n *storage.Needle, fileName string, w http.ResponseWriter, r *http.Request) (processed bool) {
	if !n.IsChunkedManifest() {
		return false
	}

	chunkManifest, e := operation.LoadChunkManifest(n.Data, n.IsGzipped())
	if e != nil {
		glog.V(0).Infof("load chunked manifest (%s) error: %v", r.URL.Path, e)
		return false
	}
	if fileName == "" && chunkManifest.Name != "" {
		fileName = chunkManifest.Name
	}
	mType := ""
	if chunkManifest.Mime != "" {
		mt := chunkManifest.Mime
		if !strings.HasPrefix(mt, "application/octet-stream") {
			mType = mt
		}
	}

	w.Header().Set("X-File-Store", "chunked")

	chunkedFileReader := &operation.ChunkedFileReader{
		Manifest: chunkManifest,
		Master:   vs.GetMasterNode(),
	}
	defer chunkedFileReader.Close()
	if e := writeResponseContent(fileName, mType, chunkedFileReader, w, r); e != nil {
		glog.V(2).Infoln("response write error:", e)
	}
	return true
}

func (vs *VolumeServer) DeleteHandler(w http.ResponseWriter, r *http.Request) {
	n := new(storage.Needle)
	vid, fid, _, _, _ := parseURLPath(r.URL.Path)
	volumeId, _ := storage.NewVolumeId(vid)
	n.ParsePath(fid)

	glog.V(2).Infoln("deleting", n)

	cookie := n.Cookie

	_, ok := vs.store.ReadVolumeNeedle(volumeId, n)
	if ok != nil {
		m := make(map[string]uint32)
		m["size"] = 0
		writeJsonQuiet(w, r, http.StatusNotFound, m)
		return
	}
	defer n.ReleaseMemory()

	if n.Cookie != cookie {
		glog.V(0).Infoln("delete", r.URL.Path, "with unmaching cookie from ", r.RemoteAddr, "agent", r.UserAgent())
		writeJsonError(w, r, http.StatusBadRequest, errors.New("File Random Cookie does not match."))
		return
	}

	count := int64(n.Size)

	if n.IsChunkedManifest() {
		chunkManifest, e := operation.LoadChunkManifest(n.Data, n.IsGzipped())
		if e != nil {
			writeJsonError(w, r, http.StatusInternalServerError, fmt.Errorf("Load chunks manifest error: %v", e))
			return
		}
		// make sure all chunks had deleted before delete manifest
		if e := chunkManifest.DeleteChunks(vs.GetMasterNode()); e != nil {
			writeJsonError(w, r, http.StatusInternalServerError, fmt.Errorf("Delete chunks error: %v", e))
			return
		}
		count = chunkManifest.Size
	}

	_, err := topology.ReplicatedDelete(vs.GetMasterNode(), vs.store, volumeId, n, r)

	if err == nil {
		m := make(map[string]int64)
		m["size"] = count
		writeJsonQuiet(w, r, http.StatusAccepted, m)
	} else {
		writeJsonError(w, r, http.StatusInternalServerError, fmt.Errorf("Deletion Failed: %v", err))
	}

}

func ReplicatedWrite(masterNode string, s *storage.Store,
	volumeId storage.VolumeId, needle *storage.Needle,
	r *http.Request) (size uint32, errorStatus string) {

	//check JWT
	jwt := security.GetJwt(r)

	ret, err := s.Write(volumeId, needle)
	needToReplicate := !s.HasVolume(volumeId)
	if err != nil {
		errorStatus = "Failed to write to local disk (" + err.Error() + ")"
	} else if ret > 0 {
		needToReplicate = needToReplicate || s.GetVolume(volumeId).NeedToReplicate()
	} else {
		errorStatus = "Failed to write to local disk"
	}
	if !needToReplicate && ret > 0 {
		needToReplicate = s.GetVolume(volumeId).NeedToReplicate()
	}
	if needToReplicate { //send to other replica locations
		if r.FormValue("type") != "replicate" {

			if err = distributedOperation(masterNode, s, volumeId, func(location operation.Location) error {
				u := url.URL{
					Scheme: "http",
					Host:   location.Url,
					Path:   r.URL.Path,
				}
				q := url.Values{
					"type": {"replicate"},
				}
				if needle.LastModified > 0 {
					q.Set("ts", strconv.FormatUint(needle.LastModified, 10))
				}
				if needle.IsChunkedManifest() {
					q.Set("cm", "true")
				}
				u.RawQuery = q.Encode()
				_, err := operation.Upload(u.String(),
					string(needle.Name), bytes.NewReader(needle.Data), needle.IsGzipped(), string(needle.Mime),
					jwt)
				return err
			}); err != nil {
				ret = 0
				errorStatus = fmt.Sprintf("Failed to write to replicas for volume %d: %v", volumeId, err)
			}
		}
	}
	size = ret
	return
}

func ReplicatedWrite(masterNode string, s *storage.Store,
	volumeId storage.VolumeId, needle *storage.Needle,
	r *http.Request) (size uint32, errorStatus string) {
	//check JWT
	jwt := security.GetJwt(r)
	defer func() {
		if errorStatus == "" {
			return
		}
		ReplicatedDelete(masterNode, s, volumeId, needle, r)
	}()
	ret, err := s.Write(volumeId, needle)
	if err != nil {
		errorStatus = "Failed to write to local disk (" + err.Error() + ")"
	} else if ret <= 0 {
		errorStatus = "Failed to write to local disk"
	}
	//send to other replica locations
	if r.FormValue("type") != "replicate" {
		repWrite := func(location operation.Location) bool {
			args := url.Values{
				"type": {"replicate"},
			}
			if needle.LastModified > 0 {
				args.Set("ts", strconv.FormatUint(needle.LastModified, 10))
			}
			if needle.IsChunkedManifest() {
				args.Set("cm", "true")
			}

			u := util.MkUrl(location.Url, r.URL.Path, args)
			glog.V(4).Infoln("write replication to", u)
			_, err := operation.Upload(u,
				string(needle.Name), bytes.NewReader(needle.Data), needle.IsGzipped(), string(needle.Mime),
				jwt)
			if err != nil {
				glog.V(0).Infof("write replication to %s err, %v", u, err)
			}
			return err == nil
		}
		if !distributedOperation(masterNode, s, volumeId, repWrite) {
			ret = 0
			errorStatus = "Failed to write to replicas for volume " + volumeId.String()
		}
	}

	size = ret
	return
}

//Experts only: takes multiple fid parameters. This function does not propagate deletes to replicas.
func (vs *VolumeServer) batchDeleteHandler(w http.ResponseWriter, r *http.Request) {
	r.ParseForm()
	var ret []operation.DeleteResult
	for _, fid := range r.Form["fid"] {
		vid, id_cookie, err := operation.ParseFileId(fid)
		if err != nil {
			ret = append(ret, operation.DeleteResult{
				Fid:    fid,
				Status: http.StatusBadRequest,
				Error:  err.Error()})
			continue
		}
		n := new(storage.Needle)
		volumeId, _ := storage.NewVolumeId(vid)
		n.ParsePath(id_cookie)
		glog.V(4).Infoln("batch deleting", n)
		cookie := n.Cookie
		if _, err := vs.store.ReadVolumeNeedle(volumeId, n); err != nil {
			ret = append(ret, operation.DeleteResult{
				Fid:    fid,
				Status: http.StatusNotFound,
				Error:  err.Error(),
			})
			continue
		}

		if n.IsChunkedManifest() {
			ret = append(ret, operation.DeleteResult{
				Fid:    fid,
				Status: http.StatusNotAcceptable,
				Error:  "ChunkManifest: not allowed in batch delete mode.",
			})
			n.ReleaseMemory()
			continue
		}

		if n.Cookie != cookie {
			ret = append(ret, operation.DeleteResult{
				Fid:    fid,
				Status: http.StatusBadRequest,
				Error:  "File Random Cookie does not match.",
			})
			glog.V(0).Infoln("deleting", fid, "with unmaching cookie from ", r.RemoteAddr, "agent", r.UserAgent())
			n.ReleaseMemory()
			return
		}
		if size, err := vs.store.Delete(volumeId, n); err != nil {
			ret = append(ret, operation.DeleteResult{
				Fid:    fid,
				Status: http.StatusInternalServerError,
				Error:  err.Error()},
			)
		} else {
			ret = append(ret, operation.DeleteResult{
				Fid:    fid,
				Status: http.StatusAccepted,
				Size:   int(size)},
			)
		}
		n.ReleaseMemory()
	}

	writeJsonQuiet(w, r, http.StatusAccepted, ret)
}